Add configs for reusing specs from environments and enable reuse from included concrete environments by default

lib/spack/spack/environment/__init__.py

@@ -473,13 +473,15 @@
     active_environment,
     all_environment_names,
     all_environments,
+    as_env_dir,
     create,
     create_in_dir,
     deactivate,
     default_manifest_yaml,
     default_view_name,
     display_specs,
     environment_dir_from_name,
+    environment_from_name_or_dir,
     exists,
     initialize_environment_dir,
     installed_specs,
@@ -507,13 +509,15 @@
     "active_environment",
     "all_environment_names",
     "all_environments",
+    "as_env_dir",
     "create",
     "create_in_dir",
     "deactivate",
     "default_manifest_yaml",
     "default_view_name",
     "display_specs",
     "environment_dir_from_name",
+    "environment_from_name_or_dir",
     "exists",
     "initialize_environment_dir",
     "installed_specs",

lib/spack/spack/environment/environment.py

@@ -277,6 +277,22 @@ def is_env_dir(path):
     return os.path.isdir(path) and os.path.exists(os.path.join(path, manifest_name))
 
 
+def as_env_dir(name_or_dir):
+    """Translate an environment name or directory to the environment directory"""
+    if is_env_dir(name_or_dir):
+        return name_or_dir
+    else:
+        validate_env_name(name_or_dir)
+        if not exists(name_or_dir):
+            raise SpackEnvironmentError("no such environment '%s'" % name_or_dir)
+        return root(name_or_dir)
+
+
+def environment_from_name_or_dir(name_or_dir):
+    """Get an environment with the supplied name."""
+    return Environment(as_env_dir(name_or_dir))
+
+
 def read(name):
     """Get an environment with the supplied name."""
     validate_env_name(name)
@@ -1506,6 +1522,7 @@ def _get_specs_to_concretize(
         # Exit early if the set of concretized specs is the set of user specs
         new_user_specs = set(self.user_specs) - set(self.concretized_user_specs)
         kept_user_specs = set(self.user_specs) & set(self.concretized_user_specs)
+        kept_user_specs |= set(self.included_user_specs)
         if not new_user_specs:
             return new_user_specs, kept_user_specs, []
 
@@ -1552,7 +1569,10 @@ def _concretize_together_where_possible(
             abstract = old_concrete_to_abstract.get(abstract, abstract)
             if abstract in new_user_specs:
                 result.append((abstract, concrete))
-            self._add_concrete_spec(abstract, concrete)
+
+            # Only add to the environment if it's from this environment (not just included)
+            if abstract in self.user_specs:
+                self._add_concrete_spec(abstract, concrete)
 
         return result
 
@@ -1595,7 +1615,9 @@ def _concretize_together(
         ordered_user_specs = list(new_user_specs) + list(kept_user_specs)
         concretized_specs = [x for x in zip(ordered_user_specs, concrete_specs)]
         for abstract, concrete in concretized_specs:
-            self._add_concrete_spec(abstract, concrete)
+            # Don't add if it's just included
+            if abstract in self.user_specs:
+                self._add_concrete_spec(abstract, concrete)
 
         # zip truncates the longer list, which is exactly what we want here
         return list(zip(new_user_specs, concrete_specs))

lib/spack/spack/schema/concretizer.py

@@ -32,8 +32,23 @@
                                     "type": "object",
                                     "properties": {
                                         "type": {
-                                            "type": "string",
-                                            "enum": ["local", "buildcache", "external"],
+                                            "oneOf": [
+                                                {
+                                                    "type": "string",
+                                                    "enum": [
+                                                        "local",
+                                                        "buildcache",
+                                                        "environment",
+                                                        "external",
+                                                    ],
+                                                },
+                                                {
+                                                    "type": "object",
+                                                    "properties": {
+                                                        "environment": {"type": "string"}
+                                                    },
+                                                },
+                                            ]
                                         },
                                         "include": LIST_OF_SPECS,
                                         "exclude": LIST_OF_SPECS,

lib/spack/spack/schema/env.py

@@ -19,6 +19,8 @@
 #: Top level key in a manifest file
 TOP_LEVEL_KEY = "spack"
 
+include_concrete = {"type": "array", "default": [], "items": {"type": "string"}}
+
 properties: Dict[str, Any] = {
     "spack": {
         "type": "object",
@@ -31,7 +33,7 @@
             {
                 "include": {"type": "array", "default": [], "items": {"type": "string"}},
                 "specs": spec_list_schema,
-                "include_concrete": {"type": "array", "default": [], "items": {"type": "string"}},
+                "include_concrete": include_concrete,
             },
         ),
     }

lib/spack/spack/solver/asp.py

@@ -2616,6 +2616,7 @@ def setup(
                 )
                 for name, info in env.dev_specs.items()
             )
+
         specs = tuple(specs)  # ensure compatible types to add
 
         self.gen.h1("Reusable concrete specs")
@@ -3966,22 +3967,45 @@ def selected_specs(self) -> List[spack.spec.Spec]:
         return [s for s in self.factory() if self.is_selected(s)]
 
     @staticmethod
-    def from_store(configuration, include, exclude) -> "SpecFilter":
+    def from_store(configuration, *, include, exclude) -> "SpecFilter":
         """Constructs a filter that takes the specs from the current store."""
         packages = _external_config_with_implicit_externals(configuration)
         is_reusable = functools.partial(_is_reusable, packages=packages, local=True)
         factory = functools.partial(_specs_from_store, configuration=configuration)
         return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
 
     @staticmethod
-    def from_buildcache(configuration, include, exclude) -> "SpecFilter":
+    def from_buildcache(configuration, *, include, exclude) -> "SpecFilter":
         """Constructs a filter that takes the specs from the configured buildcaches."""
         packages = _external_config_with_implicit_externals(configuration)
         is_reusable = functools.partial(_is_reusable, packages=packages, local=False)
         return SpecFilter(
             factory=_specs_from_mirror, is_usable=is_reusable, include=include, exclude=exclude
         )
 
+    @staticmethod
+    def from_environment(configuration, *, include, exclude, env) -> "SpecFilter":
+        packages = _external_config_with_implicit_externals(configuration)
+        is_reusable = functools.partial(_is_reusable, packages=packages, local=True)
+        factory = functools.partial(_specs_from_environment, env=env)
+        return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
+
+    @staticmethod
+    def from_environment_included_concrete(
+        configuration,
+        *,
+        include: List[str],
+        exclude: List[str],
+        env: ev.Environment,
+        included_concrete: str,
+    ) -> "SpecFilter":
+        packages = _external_config_with_implicit_externals(configuration)
+        is_reusable = functools.partial(_is_reusable, packages=packages, local=True)
+        factory = functools.partial(
+            _specs_from_environment_included_concrete, env=env, included_concrete=included_concrete
+        )
+        return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
+
 
 def _specs_from_store(configuration):
     store = spack.store.create(configuration)
@@ -3999,6 +4023,23 @@ def _specs_from_mirror():
         return []
 
 
+def _specs_from_environment(env):
+    """Return all concrete specs from the environment. This includes all included concrete"""
+    if env:
+        return [concrete for _, concrete in env.concretized_specs()]
+    else:
+        return []
+
+
+def _specs_from_environment_included_concrete(env, included_concrete):
+    """Return only concrete specs from the environment included from the included_concrete"""
+    if env:
+        assert included_concrete in env.included_concrete_envs
+        return [concrete for concrete in env.included_specs_by_hash[included_concrete].values()]
+    else:
+        return []
+
+
 class ReuseStrategy(enum.Enum):
     ROOTS = enum.auto()
     DEPENDENCIES = enum.auto()
@@ -4028,6 +4069,12 @@ def __init__(self, configuration: spack.config.Configuration) -> None:
                     SpecFilter.from_buildcache(
                         configuration=self.configuration, include=[], exclude=[]
                     ),
+                    SpecFilter.from_environment(
+                        configuration=self.configuration,
+                        include=[],
+                        exclude=[],
+                        env=ev.active_environment(),  # includes all concrete includes
+                    ),
                 ]
             )
         else:
@@ -4042,7 +4089,46 @@ def __init__(self, configuration: spack.config.Configuration) -> None:
             for source in reuse_yaml.get("from", default_sources):
                 include = source.get("include", default_include)
                 exclude = source.get("exclude", default_exclude)
-                if source["type"] == "local":
+                if isinstance(source["type"], dict):
+                    env_dir = ev.as_env_dir(source["type"].get("environment"))
+                    active_env = ev.active_environment()
+                    if active_env and env_dir in active_env.included_concrete_envs:
+                        # If environment is included as a concrete environment, use the local copy
+                        # of specs in the active environment.
+                        # note: included concrete environments are only updated at concretization
+                        #       time, and reuse needs to matchthe included specs.
+                        self.reuse_sources.append(
+                            SpecFilter.from_environment_included_concrete(
+                                self.configuration,
+                                include=include,
+                                exclude=exclude,
+                                env=active_env,
+                                included_concrete=env_dir,
+                            )
+                        )
+                    else:
+                        # If the environment is not included as a concrete environment, use the
+                        # current specs from its lockfile.
+                        self.reuse_sources.append(
+                            SpecFilter.from_environment(
+                                self.configuration,
+                                include=include,
+                                exclude=exclude,
+                                env=ev.environment_from_name_or_dir(env_dir),
+                            )
+                        )
+                elif source["type"] == "environment":
+                    # reusing from the current environment implicitly reuses from all of the
+                    # included concrete environments
+                    self.reuse_sources.append(
+                        SpecFilter.from_environment(
+                            self.configuration,
+                            include=include,
+                            exclude=exclude,
+                            env=ev.active_environment(),
+                        )
+                    )
+                elif source["type"] == "local":
                     self.reuse_sources.append(
                         SpecFilter.from_store(self.configuration, include=include, exclude=exclude)
                     )