Merge pull request #342 from pyiron/refactor

Refactor interface

Author: jan-janssen

pysqa/utils/__init__.py pysqa/base/__init__.py

@@ -2,13 +2,58 @@
 from typing import List, Optional, Tuple, Union
 
 import pandas
+import yaml
 from jinja2 import Template
 from jinja2.exceptions import TemplateSyntaxError
 
-from pysqa.utils.core import QueueAdapterCore
-from pysqa.utils.execute import execute_command
-from pysqa.utils.queues import Queues
-from pysqa.utils.validate import value_error_if_none, value_in_range
+from pysqa.base.core import QueueAdapterCore, execute_command
+from pysqa.base.validate import check_queue_parameters, value_error_if_none
+
+
+class Queues(object):
+    """
+    Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in
+    interactive environments like jupyter.
+    """
+
+    def __init__(self, list_of_queues: List[str]):
+        """
+        Initialize the Queues object.
+
+        Args:
+            list_of_queues (List[str]): A list of queue names.
+
+        """
+        self._list_of_queues = list_of_queues
+
+    def __getattr__(self, item: str) -> str:
+        """
+        Get the queue name.
+
+        Args:
+            item (str): The name of the queue.
+
+        Returns:
+            str: The name of the queue.
+
+        Raises:
+            AttributeError: If the queue name is not in the list of queues.
+
+        """
+        if item in self._list_of_queues:
+            return item
+        else:
+            raise AttributeError
+
+    def __dir__(self) -> List[str]:
+        """
+        Get the list of queues.
+
+        Returns:
+            List[str]: The list of queues.
+
+        """
+        return self._list_of_queues
 
 
 class QueueAdapterWithConfig(QueueAdapterCore):
@@ -162,18 +207,12 @@ def check_queue_parameters(
         """
         if active_queue is None:
             active_queue = self._config["queues"][queue]
-        cores = value_in_range(
-            value=cores,
-            value_min=active_queue["cores_min"],
-            value_max=active_queue["cores_max"],
-        )
-        run_time_max = value_in_range(
-            value=run_time_max, value_max=active_queue["run_time_max"]
-        )
-        memory_max = value_in_range(
-            value=memory_max, value_max=active_queue["memory_max"]
+        return check_queue_parameters(
+            active_queue=active_queue,
+            cores=cores,
+            run_time_max=run_time_max,
+            memory_max=memory_max,
         )
-        return cores, run_time_max, memory_max
 
     def _job_submission_template(
         self,
@@ -270,3 +309,17 @@ def _load_templates(queue_lst_dict: dict, directory: str = ".") -> None:
                             + error.message,
                             lineno=error.lineno,
                         )
+
+
+def read_config(file_name: str = "queue.yaml") -> dict:
+    """
+    Read and parse a YAML configuration file.
+
+    Args:
+        file_name (str): The name of the YAML file to read.
+
+    Returns:
+        dict: The parsed configuration as a dictionary.
+    """
+    with open(file_name, "r") as f:
+        return yaml.load(f, Loader=yaml.FullLoader)

pysqa/utils/basic.py pysqa/base/config.py

@@ -1,12 +1,12 @@
 import getpass
 import importlib
 import os
+import subprocess
 from typing import List, Optional, Tuple, Union
 
 import pandas
 from jinja2 import Template
 
-from pysqa.utils.execute import execute_command
 from pysqa.wrapper.abstract import SchedulerCommands
 
 queue_type_dict = {
@@ -45,6 +45,46 @@
 }
 
 
+def execute_command(
+    commands: str,
+    working_directory: Optional[str] = None,
+    split_output: bool = True,
+    shell: bool = False,
+    error_filename: str = "pysqa.err",
+) -> Union[str, List[str]]:
+    """
+    A wrapper around the subprocess.check_output function.
+
+    Args:
+        commands (str): The command(s) to be executed on the command line
+        working_directory (str, optional): The directory where the command is executed. Defaults to None.
+        split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True.
+        shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False.
+        error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err".
+
+    Returns:
+        Union[str, List[str]]: Output of the shell command either as a string or as a list of strings
+    """
+    if shell and isinstance(commands, list):
+        commands = " ".join(commands)
+    try:
+        out = subprocess.check_output(
+            commands,
+            cwd=working_directory,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            shell=not isinstance(commands, list),
+        )
+    except subprocess.CalledProcessError as e:
+        with open(os.path.join(working_directory, error_filename), "w") as f:
+            print(e.stdout, file=f)
+        out = None
+    if out is not None and split_output:
+        return out.split("\n")
+    else:
+        return out
+
+
 def get_queue_commands(queue_type: str) -> Union[SchedulerCommands, None]:
     """
     Load queuing system commands class

pysqa/utils/core.py pysqa/base/core.py

@@ -1,5 +1,36 @@
 import re
-from typing import Union
+from typing import Optional, Tuple, Union
+
+
+def check_queue_parameters(
+    active_queue: Optional[dict] = None,
+    cores: int = 1,
+    run_time_max: Optional[int] = None,
+    memory_max: Optional[int] = None,
+) -> Tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]:
+    """
+    Check the parameters of a queue.
+
+    Args:
+
+        cores (int, optional): The number of cores. Defaults to 1.
+        run_time_max (int, optional): The maximum run time. Defaults to None.
+        memory_max (int, optional): The maximum memory. Defaults to None.
+        active_queue (dict, optional): The active queue. Defaults to None.
+
+    Returns:
+        list: A list of queue parameters [cores, run_time_max, memory_max].
+    """
+    cores = value_in_range(
+        value=cores,
+        value_min=active_queue["cores_min"],
+        value_max=active_queue["cores_max"],
+    )
+    run_time_max = value_in_range(
+        value=run_time_max, value_max=active_queue["run_time_max"]
+    )
+    memory_max = value_in_range(value=memory_max, value_max=active_queue["memory_max"])
+    return cores, run_time_max, memory_max
 
 
 def value_error_if_none(value: str) -> None:

pysqa/utils/validate.py pysqa/base/validate.py

@@ -4,8 +4,8 @@
 import sys
 from typing import Optional
 
+from pysqa.base.core import execute_command
 from pysqa.queueadapter import QueueAdapter
-from pysqa.utils.execute import execute_command
 
 
 def command_line(

pysqa/cmd.py

@@ -2,8 +2,8 @@
 
 import pandas
 
-from pysqa.utils.basic import QueueAdapterWithConfig
-from pysqa.utils.execute import execute_command
+from pysqa.base.config import QueueAdapterWithConfig
+from pysqa.base.core import execute_command
 
 
 class ModularQueueAdapter(QueueAdapterWithConfig):

pysqa/ext/modular.py

@@ -8,8 +8,8 @@
 import paramiko
 from tqdm import tqdm
 
-from pysqa.utils.basic import QueueAdapterWithConfig
-from pysqa.utils.execute import execute_command
+from pysqa.base.config import QueueAdapterWithConfig
+from pysqa.base.core import execute_command
 
 
 class RemoteQueueAdapter(QueueAdapterWithConfig):

pysqa/ext/remote.py

@@ -3,10 +3,9 @@
 
 import pandas
 
+from pysqa.base.config import QueueAdapterWithConfig, read_config
+from pysqa.base.core import execute_command
 from pysqa.ext.modular import ModularQueueAdapter
-from pysqa.utils.basic import QueueAdapterWithConfig
-from pysqa.utils.config import read_config
-from pysqa.utils.execute import execute_command
 
 
 class QueueAdapter(object):

pysqa/queueadapter.py

@@ -5,8 +5,8 @@
 import unittest
 from jinja2.exceptions import TemplateSyntaxError
 from pysqa import QueueAdapter
-from pysqa.utils.basic import QueueAdapterWithConfig
-from pysqa.utils.validate import value_in_range
+from pysqa.base.config import QueueAdapterWithConfig
+from pysqa.base.validate import value_in_range
 
 __author__ = "Jan Janssen"
 __copyright__ = "Copyright 2019, Jan Janssen"

pysqa/utils/config.py

@@ -1,6 +1,6 @@
 import os
 import unittest
-from pysqa.utils.execute import execute_command
+from pysqa.base.core import execute_command
 
 
 class TestExecuteCommand(unittest.TestCase):

pysqa/utils/execute.py

@@ -6,7 +6,7 @@
 import unittest
 import getpass
 from pysqa import QueueAdapter
-from pysqa.utils.validate import value_in_range
+from pysqa.base.validate import value_in_range
 
 try:
     import defusedxml.ElementTree as ETree