Add directory support for generator to_configure parameter

smartsim/entity/files.py

@@ -24,6 +24,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import os
 from os import path
 
 from ..error import SSConfigError
@@ -63,6 +64,7 @@ def __init__(self, tagged, copy, symlink):
         self.tagged = tagged
         self.copy = copy
         self.link = symlink
+        self.tagged_hierarchy = None
         self._check_files()
 
     def _check_files(self):
@@ -78,12 +80,9 @@ def _check_files(self):
         self.copy = self._type_check_files(self.copy, "Copyable")
         self.link = self._type_check_files(self.link, "Symlink")
 
-        # check the paths provided by the user and ensure
-        # that no directories were provided as tagged files
-        for i in range(len(self.tagged)):
-            self.tagged[i] = self._check_path(self.tagged[i])
-            if path.isdir(self.tagged[i]):
-                raise SSConfigError("Directories cannot be listed in tagged files")
+        self.tagged_hierarchy = TaggedFilesHierarchy.from_list_paths(
+            self.tagged, dir_contents_to_base=True
+        )
 
         for i in range(len(self.copy)):
             self.copy[i] = self._check_path(self.copy[i])
@@ -114,7 +113,8 @@ def _type_check_files(self, file_list, file_type):
                     raise TypeError(f"Not all {file_type} files were of type str")
         return file_list
 
-    def _check_path(self, file_path):
+    @staticmethod
+    def _check_path(file_path):
         """Given a user provided path-like str, find the actual path to
            the directory or file and create a full path.
 
@@ -130,3 +130,122 @@ def _check_path(self, file_path):
         if path.isdir(full_path):
             return full_path
         raise SSConfigError(f"File or Directory {file_path} not found")
+
+
+class TaggedFilesHierarchy:
+    """The TaggedFilesHierarchy class maintains a list of files and a list of
+    child TaggedFilesHierarchy instances. Each instance describes how a
+    directory structure of tagged files.
+
+    TaggedFilesHierarchy represents a directory containing potentially tagged
+    files and subdirectories.
+
+    TaggedFilesHierarchy.base is the directory path from the the root of the
+    file structure
+    E.g.
+    TaggedFilesHierarchy.base = ""            => ./
+    TaggedFilesHierarchy.base = "dir_a"       => ./dir_a
+    TaggedFilesHierarchy.base = "dir_a/dir_b" => ./dir_a/dir_b
+
+    TaggedFilesHierarchy.files is a collection of paths to files that need
+    to be copied to directory that the TaggedFilesHierarchy represents
+
+    TaggedFilesHierarchy.dirs is a collection of child TaggedFilesHierarchy,
+    each with its own files and dirs, representing a directory that needs to
+    be created within the current directory represented by the current
+    TaggedFilesHierarchy
+
+    By performing a depth first search over the entire hierarchy starting at
+    the root (TaggedFilesHierarchy.base = ""), one could reconstruct the
+    tagged file directory structure with relative ease at any location deemed
+    necessary.
+    """
+
+    def __init__(self, base=""):
+        """Initialize a TaggedFilesHierarchy
+
+        :param base: pathlike string specifing the generated directory
+                     files are located
+        :type base: str, optional
+        """
+        self.base = base
+        self.files = set()
+        self.dirs = set()
+
+    @classmethod
+    def from_list_paths(cls, path_list, dir_contents_to_base=False):
+        """Given a list of absolute paths to files and dirs, create and return
+        a TaggedFilesHierarchy instance representing the file hierarchy of
+        tagged files. All files in the path list will be placed in the base of
+        the file hierarchy.
+
+        :param path_list: list of absolute paths to tagged files or dirs
+                          containing tagged files
+        :type path_list: list[str]
+        :param dir_contents_to_base: When a top level dir is encountered, if
+                                     this value is truthy, files in the dir are
+                                     put into the base hierarchy level.
+                                     Otherwise, a new sub level is created for
+                                     the dir
+        :type dir_contents_to_base: bool
+        :return: A built tagged file hierarchy for the given files
+        :rtype: TaggedFilesHierarchy
+        """
+        tagged_file_hierarchy = cls()
+        if dir_contents_to_base:
+            new_paths = []
+            for path in path_list:
+                if os.path.isdir(path):
+                    new_paths += [os.path.join(path, file) for file in os.listdir(path)]
+                else:
+                    new_paths.append(path)
+            path_list = new_paths
+        tagged_file_hierarchy._add_paths(path_list)
+        return tagged_file_hierarchy
+
+    def _add_file(self, file):
+        """Add a file to the current level in the file hierarchy
+
+        :param file: absoute path to a file to add to the hierarchy
+        :type file: str
+        """
+        self.files.add(file)
+
+    def _add_dir(self, dir):
+        """Add a dir contianing tagged files by creating a new sub level in the
+        tagged file hierarchy. All paths within the directroy are added to the
+        the new level sub level tagged file hierarchy
+
+        :param dir: absoute path to a dir to add to the hierarchy
+        :type dir: str
+        """
+        tagged_file_hierarchy = TaggedFilesHierarchy(
+            path.join(self.base, path.basename(dir))
+        )
+        tagged_file_hierarchy._add_paths(
+            [path.join(dir, file) for file in os.listdir(dir)]
+        )
+        self.dirs.add(tagged_file_hierarchy)
+
+    def _add_paths(self, paths):
+        """Takes a list of paths and iterates over it, determining if each
+        path is to a file or a dir and then appropriatly adding it to the
+        TaggedFilesHierarchy.
+
+        :param paths: list of paths to files or dirs to add to the hierarchy
+        :type paths: list[str]
+        :raises SSConfigError: if link to dir is found or path does not exist
+        """
+        for path in paths:
+            path = os.path.abspath(path)
+            if os.path.isdir(path):
+                if os.path.islink(path):
+                    raise SSConfigError(
+                        "Tagged directories and thier subdirectories cannot be links"
+                        + " to prevent circular directory structures"
+                    )
+                self._add_dir(path)
+            elif os.path.isfile(path):
+                self._add_file(path)
+            else:
+                raise SSConfigError(f"File or Directory {path} not found")

smartsim/experiment.py

@@ -24,8 +24,8 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import time
 import os.path as osp
+import time
 from os import getcwd
 from pprint import pformat
 

smartsim/generation/generator.py

@@ -211,10 +211,25 @@ def _write_tagged_entity_files(self, entity):
         """
         if entity.files:
             to_write = []
-            for tagged_file in entity.files.tagged:
-                dst_path = path.join(entity.path, path.basename(tagged_file))
-                shutil.copyfile(tagged_file, dst_path)
-                to_write.append(dst_path)
+
+            def _build_tagged_files(tagged):
+                """Using a TaggedFileHierarchy, reproduce the tagged file
+                directory structure
+
+                :param tagged: a TaggedFileHierarchy to be built as a
+                               directory structure
+                :type tagged: TaggedFilesHierarchy
+                """
+                for file in tagged.files:
+                    dst_path = path.join(entity.path, tagged.base, path.basename(file))
+                    shutil.copyfile(file, dst_path)
+                    to_write.append(dst_path)
+
+                for dir in tagged.dirs:
+                    mkdir(path.join(entity.path, tagged.base, path.basename(dir.base)))
+                    _build_tagged_files(dir)
+
+            _build_tagged_files(entity.files.tagged_hierarchy)
 
             # write in changes to configurations
             if isinstance(entity, Model):

smartsim/settings/cobaltSettings.py

@@ -50,14 +50,15 @@ def __init__(
         :param batch_args: extra batch arguments, defaults to None
         :type batch_args: dict[str, str], optional
         """
-        super().__init__("qsub",
-                         batch_args=batch_args,
-                         nodes=nodes, 
-                         account=account, 
-                         queue=queue, 
-                         time=time,  
-                         **kwargs)
-
+        super().__init__(
+            "qsub",
+            batch_args=batch_args,
+            nodes=nodes,
+            account=account,
+            queue=queue,
+            time=time,
+            **kwargs,
+        )
 
     def set_walltime(self, walltime):
         """Set the walltime of the job

smartsim/settings/lsfSettings.py

@@ -125,7 +125,7 @@ def set_tasks_per_rs(self, tasks_per_rs):
 
     def set_tasks_per_node(self, tasks_per_node):
         """Set the number of tasks per resource set.
-         
+
         This function is an alias for `set_tasks_per_rs`.
 
         :param tasks_per_node: number of tasks per resource set
@@ -138,13 +138,13 @@ def set_hostlist(self, host_list):
 
         This function is only available to unify LSFSettings
         to other WLM settings classes.
-        
+
         """
         pass
 
     def set_cpus_per_task(self, cpus_per_task):
         """Set the number of cpus per tasks.
-         
+
         This function is an alias for `set_cpus_per_rs`.
 
         :param cpus_per_task: number of cpus per resource set
@@ -335,13 +335,15 @@ def __init__(
             kwargs.pop("account", None)
         else:
             project = kwargs.pop("account", None)
-
-        super().__init__("bsub",
-                         batch_args=batch_args,
-                         nodes=nodes,
-                         account=project,
-                         time=time,
-                         **kwargs)
+
+        super().__init__(
+            "bsub",
+            batch_args=batch_args,
+            nodes=nodes,
+            account=project,
+            time=time,
+            **kwargs,
+        )
 
         if smts:
             self.set_smts(smts)

smartsim/settings/pbsSettings.py

@@ -65,13 +65,15 @@ def __init__(
         :type batch_args: dict[str, str], optional
         """
 
-        super().__init__("qsub", 
-                         batch_args=batch_args,
-                         nodes=nodes, 
-                         account=account, 
-                         queue=queue, 
-                         time=time, 
-                         **kwargs)
+        super().__init__(
+            "qsub",
+            batch_args=batch_args,
+            nodes=nodes,
+            account=account,
+            queue=queue,
+            time=time,
+            **kwargs,
+        )
         self.resources = init_default({}, resources, dict)
 
         self._ncpus = ncpus

smartsim/settings/settings.py

@@ -68,7 +68,12 @@ def create_batch_settings(
     try:
         batch_class = by_launcher[launcher]
         batch_settings = batch_class(
-            nodes=nodes, time=time, batch_args=batch_args, queue=queue, account=account, **kwargs
+            nodes=nodes,
+            time=time,
+            batch_args=batch_args,
+            queue=queue,
+            account=account,
+            **kwargs,
         )
         return batch_settings
 

smartsim/settings/slurmSettings.py

@@ -224,12 +224,14 @@ def __init__(self, nodes=None, time="", account=None, batch_args=None, **kwargs)
         :param batch_args: extra batch arguments, defaults to None
         :type batch_args: dict[str, str], optional
         """
-        super().__init__("sbatch",
-                         batch_args=batch_args,
-                         nodes=nodes, 
-                         account=account, 
-                         time=time,  
-                         **kwargs)
+        super().__init__(
+            "sbatch",
+            batch_args=batch_args,
+            nodes=nodes,
+            account=account,
+            time=time,
+            **kwargs,
+        )
 
     def set_walltime(self, walltime):
         """Set the walltime of the job

tests/on_wlm/test_launch_orc_slurm.py

@@ -72,7 +72,7 @@ def test_incoming_entities(fileutils, wlmutils):
     launcher = wlmutils.get_test_launcher()
     if launcher != "slurm":
         pytest.skip("Test only runs on systems with Slurm as WLM")
-        
+
     exp_name = "test-incoming-entities"
     exp = Experiment(exp_name, launcher=wlmutils.get_test_launcher())
     test_dir = fileutils.make_test_dir(exp_name)

tests/on_wlm/test_simple_base_settings_on_wlm.py

@@ -25,7 +25,9 @@
 def test_simple_model_on_wlm(fileutils, wlmutils):
     launcher = wlmutils.get_test_launcher()
     if launcher not in ["pbs", "slurm", "cobalt", "lsf"]:
-        pytest.skip("Test only runs on systems with LSF, PBSPro, Slurm, or Cobalt as WLM")
+        pytest.skip(
+            "Test only runs on systems with LSF, PBSPro, Slurm, or Cobalt as WLM"
+        )
 
     exp_name = "test-simplebase-settings-model-launch"
     exp = Experiment(exp_name, launcher=wlmutils.get_test_launcher())
@@ -44,7 +46,9 @@ def test_simple_model_on_wlm(fileutils, wlmutils):
 def test_simple_model_stop_on_wlm(fileutils, wlmutils):
     launcher = wlmutils.get_test_launcher()
     if launcher not in ["pbs", "slurm", "cobalt", "lsf"]:
-        pytest.skip("Test only runs on systems with LSF, PBSPro, Slurm, or Cobalt as WLM")
+        pytest.skip(
+            "Test only runs on systems with LSF, PBSPro, Slurm, or Cobalt as WLM"
+        )
 
     exp_name = "test-simplebase-settings-model-stop"
     exp = Experiment(exp_name, launcher=wlmutils.get_test_launcher())