Merge pull request #509 from pepkit/dev

Looper 2.0.0 Release

docs/changelog.md

@@ -2,6 +2,34 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [2.0.0] -- 2025-01-16
+
+This release breaks backwards compatibility for Looper versions < 2.0.0
+
+### Fixed
+- divvy init [#520](https://github.com/pepkit/looper/issues/520)
+- replaced deprecated PEPHubClient function, `_load_raw_pep` with `.load_raw_pep`
+- looper cli parameters now take priority as originally intended [#518](https://github.com/pepkit/looper/issues/518)
+- fix divvy inspect
+- remove printed dictionary at looper finish [#511](https://github.com/pepkit/looper/issues/511)
+- fix [#536](https://github.com/pepkit/looper/issues/536)
+- fix [#522](https://github.com/pepkit/looper/issues/522)
+- fix [#537](https://github.com/pepkit/looper/issues/537)
+- fix [#534](https://github.com/pepkit/looper/issues/534)
+
+### Changed
+- `--looper-config` is now `--config`, `-c`. [#455](https://github.com/pepkit/looper/issues/455)
+- A pipeline interface now consolidates a `sample_interface` and a `project_interface` [#493](https://github.com/pepkit/looper/issues/493)
+- Updated documentation for Looper 2.0.0, removing previous versions [pepspec PR #34](https://github.com/pepkit/pepspec/pull/34)
+- remove position based argument for divvy config, must use --config or run as default config
+
+### Added
+- `looper init` tutorial [#466](https://github.com/pepkit/looper/issues/466)
+- looper config allows for `pephub_path` in pipestat config section of `.looper.yaml` [#519](https://github.com/pepkit/looper/issues/519)
+- improve error messaging for bad/malformed looper configurations [#515](https://github.com/pepkit/looper/issues/515)
+- add shortform argument for --package (alias is now -p)
+
+
 ## [1.9.1] -- 2024-07-18
 
 ### Changed

looper/_version.py

@@ -1,2 +1,2 @@
-__version__ = "1.9.1"
-# You must change the version in parser = pydantic2_argparse.ArgumentParser in cli_pydantic.py!!!
+__version__ = "2.0.0"
+# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!

looper/cli_divvy.py

@@ -53,10 +53,10 @@ def add_subparser(cmd, description):
 
     for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
         sp.add_argument(
-            "config", nargs="?", default=None, help="Divvy configuration file."
+            "--config", nargs="?", default=None, help="Divvy configuration file."
         )
 
-    sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
+    sps["init"].add_argument("--config", default=None, help="Divvy configuration file.")
 
     for sp in [sps["inspect"]]:
         sp.add_argument(
@@ -124,9 +124,11 @@ def main():
         sys.exit(0)
 
     _LOGGER.debug("Divvy config: {}".format(args.config))
+
     divcfg = select_divvy_config(args.config)
+
     _LOGGER.info("Using divvy config: {}".format(divcfg))
-    dcc = ComputingConfiguration(filepath=divcfg)
+    dcc = ComputingConfiguration.from_yaml_file(filepath=divcfg)
 
     if args.command == "list":
         # Output header via logger and content via print so the user can
@@ -142,11 +144,13 @@ def main():
         for pkg_name, pkg in dcc.compute_packages.items():
             if pkg_name == args.package:
                 found = True
-                with open(pkg.submission_template, "r") as f:
+                with open(pkg["submission_template"], "r") as f:
                     print(f.read())
-                _LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
+                _LOGGER.info(
+                    "Submission command is: " + pkg["submission_command"] + "\n"
+                )
                 if pkg_name == "docker":
-                    print("Docker args are: " + pkg.docker_args)
+                    print("Docker args are: " + pkg["docker_args"])
 
         if not found:
             _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")

looper/cli_pydantic.py

@@ -26,8 +26,6 @@
 from pephubclient import PEPHubClient
 from pydantic_argparse.argparse.parser import ArgumentParser
 
-from divvy import select_divvy_config
-
 from . import __version__
 
 from .command_models.arguments import ArgumentEnum
@@ -54,9 +52,11 @@
     read_yaml_file,
     inspect_looper_config_file,
     is_PEP_file_type,
+    looper_config_tutorial,
 )
 
 from typing import List, Tuple
+from rich.console import Console
 
 
 def opt_attr_pair(name: str) -> Tuple[str, str]:
@@ -122,52 +122,56 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         sys.exit(1)
 
     if subcommand_name == "init":
-        return int(
-            not initiate_looper_config(
-                dotfile_path(),
-                subcommand_args.pep_config,
-                subcommand_args.output_dir,
-                subcommand_args.sample_pipeline_interfaces,
-                subcommand_args.project_pipeline_interfaces,
-                subcommand_args.force_yes,
+
+        console = Console()
+        console.clear()
+        console.rule(f"\n[magenta]Looper initialization[/magenta]")
+        selection = subcommand_args.generic
+        if selection is True:
+            console.clear()
+            return int(
+                not initiate_looper_config(
+                    dotfile_path(),
+                    subcommand_args.pep_config,
+                    subcommand_args.output_dir,
+                    subcommand_args.sample_pipeline_interfaces,
+                    subcommand_args.project_pipeline_interfaces,
+                    subcommand_args.force_yes,
+                )
             )
-        )
+        else:
+            console.clear()
+            return int(looper_config_tutorial())
 
     if subcommand_name == "init_piface":
         sys.exit(int(not init_generic_pipeline()))
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
-    if subcommand_args.config_file is None:
-        looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
-        try:
-            if subcommand_args.looper_config:
-                looper_config_dict = read_looper_config_file(
-                    subcommand_args.looper_config
-                )
+    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+    try:
+        if subcommand_args.config:
+            looper_config_dict = read_looper_config_file(subcommand_args.config)
+        else:
+            looper_config_dict = read_looper_dotfile()
+            _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+        cli_modifiers_dict = None
+        for looper_config_key, looper_config_item in looper_config_dict.items():
+            if looper_config_key == CLI_KEY:
+                cli_modifiers_dict = looper_config_item
             else:
-                looper_config_dict = read_looper_dotfile()
-                _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
-
-            cli_modifiers_dict = None
-            for looper_config_key, looper_config_item in looper_config_dict.items():
-                if looper_config_key == CLI_KEY:
-                    cli_modifiers_dict = looper_config_item
-                else:
-                    setattr(subcommand_args, looper_config_key, looper_config_item)
-
-        except OSError:
-            parser.print_help(sys.stderr)
+                setattr(subcommand_args, looper_config_key, looper_config_item)
+
+    except OSError as e:
+        if subcommand_args.config:
             _LOGGER.warning(
-                f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
+                f"\nLooper config file does not exist at given path {subcommand_args.config}. Use looper init to create one at {looper_cfg_path}."
             )
-            sys.exit(1)
-    else:
-        _LOGGER.warning(
-            "This PEP configures looper through the project config. This approach is deprecated and will "
-            "be removed in future versions. Please use a looper config file. For more information see "
-            "looper.databio.org/en/latest/looper-config"
-        )
+        else:
+            _LOGGER.warning(e)
+
+        sys.exit(1)
 
     subcommand_args = enrich_args_via_cfg(
         subcommand_name,
@@ -191,12 +195,12 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         subcommand_args.ignore_flags = True
 
     # Initialize project
-    if is_PEP_file_type(subcommand_args.config_file) and os.path.exists(
-        subcommand_args.config_file
+    if is_PEP_file_type(subcommand_args.pep_config) and os.path.exists(
+        subcommand_args.pep_config
     ):
         try:
             p = Project(
-                cfg=subcommand_args.config_file,
+                cfg=subcommand_args.pep_config,
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
@@ -209,14 +213,14 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         except yaml.parser.ParserError as e:
             _LOGGER.error(f"Project config parse failed -- {e}")
             sys.exit(1)
-    elif is_pephub_registry_path(subcommand_args.config_file):
+    elif is_pephub_registry_path(subcommand_args.pep_config):
         if vars(subcommand_args)[SAMPLE_PL_ARG]:
             p = Project(
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
-                project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=subcommand_args.config_file
+                project_dict=PEPHubClient().load_raw_pep(
+                    registry_path=subcommand_args.pep_config
                 ),
                 **{
                     attr: getattr(subcommand_args, attr)
@@ -252,7 +256,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
         is_pipestat_configured = (
             prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
-            if getattr(subcommand_args, "project", None)
+            if getattr(subcommand_args, "project", None) or subcommand_name == "runp"
             else prj._check_if_pipestat_configured()
         )
 
@@ -330,13 +334,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 _LOGGER.warning("No looper configuration was supplied.")
 
 
-def main(test_args=None) -> None:
+def main(test_args=None) -> dict:
     parser = pydantic_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
         description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="1.9.1",
+        version="2.0.0",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)
@@ -349,6 +353,10 @@ def main(test_args=None) -> None:
     return run_looper(args, parser, test_args=test_args)
 
 
+def main_cli() -> None:
+    main()
+
+
 def _proc_resources_spec(args):
     """
     Process CLI-sources compute setting specification. There are two sources
@@ -375,20 +383,29 @@ def _proc_resources_spec(args):
         settings_data = {}
     if not spec:
         return settings_data
-    pairs = [(kv, kv.split("=")) for kv in spec]
-    bads = []
-    for orig, pair in pairs:
-        try:
-            k, v = pair
-        except ValueError:
-            bads.append(orig)
-        else:
-            settings_data[k] = v
-    if bads:
-        raise ValueError(
-            "Could not correctly parse itemized compute specification. "
-            "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
-        )
+    if isinstance(
+        spec, str
+    ):  # compute: "partition=standard time='01-00:00:00' cores='32' mem='32000'"
+        spec = spec.split(sep=" ")
+    if isinstance(spec, list):
+        pairs = [(kv, kv.split("=")) for kv in spec]
+        bads = []
+        for orig, pair in pairs:
+            try:
+                k, v = pair
+            except ValueError:
+                bads.append(orig)
+            else:
+                settings_data[k] = v
+        if bads:
+            raise ValueError(
+                "Could not correctly parse itemized compute specification. "
+                "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
+            )
+    elif isinstance(spec, dict):
+        for key, value in spec.items():
+            settings_data[key] = value
+
     return settings_data
 
 

looper/command_models/arguments.py

@@ -162,13 +162,9 @@ class ArgumentEnum(enum.Enum):
         default=(int, None),
         description="Skip samples by numerical index",
     )
-    CONFIG_FILE = Argument(
-        name="config_file",
-        default=(str, None),
-        description="Project configuration file",
-    )
-    LOOPER_CONFIG = Argument(
-        name="looper_config",
+    CONFIG = Argument(
+        name="config",
+        alias="-c",
         default=(str, None),
         description="Looper configuration file (YAML)",
     )
@@ -188,6 +184,20 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Output directory",
     )
+    REPORT_OUTPUT_DIR = Argument(
+        name="report_dir",
+        alias="-r",
+        default=(str, None),
+        description="Set location for looper report and looper table outputs",
+    )
+
+    GENERIC = Argument(
+        name="generic",
+        alias="-g",
+        default=(bool, False),
+        description="Use generic looper config?",
+    )
+
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
         alias="-S",
@@ -232,12 +242,12 @@ class ArgumentEnum(enum.Enum):
     )
     PACKAGE = Argument(
         name="package",
+        alias="-p",
         default=(str, None),
         description="Name of computing resource package to use",
     )
     COMPUTE = Argument(
         name="compute",
-        alias="-c",
         default=(List, []),
         description="List of key-value pairs (k1=v1)",
     )

looper/command_models/commands.py

@@ -53,8 +53,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     ArgumentEnum.SKIP.value,
     ArgumentEnum.PEP_CONFIG.value,
     ArgumentEnum.OUTPUT_DIR.value,
-    ArgumentEnum.CONFIG_FILE.value,
-    ArgumentEnum.LOOPER_CONFIG.value,
+    ArgumentEnum.CONFIG.value,
     ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
     ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
     ArgumentEnum.PIPESTAT.value,
@@ -125,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 TableParser = Command(
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
-    [],
+    [
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
+    ],
 )
 
 
@@ -135,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
     ],
 )
 
@@ -188,6 +190,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PEP_CONFIG.value,
         ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
         ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.GENERIC.value,
     ],
 )