From 256d5856d4dfc3c3d0ead054166867a013420b54 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 25 Jun 2024 12:58:51 -0400
Subject: [PATCH 01/56] remove checking for config file

---
 looper/cli_pydantic.py | 52 ++++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 30 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 9e5aca16..669c7f65 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -138,39 +138,31 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
-    if subcommand_args.config_file is None:
-        looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
-        try:
-            if subcommand_args.looper_config:
-                looper_config_dict = read_looper_config_file(
-                    subcommand_args.looper_config
-                )
+    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+
+    try:
+        if subcommand_args.looper_config:
+            looper_config_dict = read_looper_config_file(subcommand_args.looper_config)
+        else:
+            looper_config_dict = read_looper_dotfile()
+            _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+        sample_modifiers_dict = None
+        cli_modifiers_dict = None
+        for looper_config_key, looper_config_item in looper_config_dict.items():
+            if looper_config_key == SAMPLE_MODS_KEY:
+                sample_modifiers_dict = looper_config_item
+            elif looper_config_key == CLI_MODS_KEY:
+                cli_modifiers_dict = looper_config_item
             else:
-                looper_config_dict = read_looper_dotfile()
-                _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
-
-            sample_modifiers_dict = None
-            cli_modifiers_dict = None
-            for looper_config_key, looper_config_item in looper_config_dict.items():
-                if looper_config_key == SAMPLE_MODS_KEY:
-                    sample_modifiers_dict = looper_config_item
-                elif looper_config_key == CLI_MODS_KEY:
-                    cli_modifiers_dict = looper_config_item
-                else:
-                    setattr(subcommand_args, looper_config_key, looper_config_item)
-
-        except OSError:
-            parser.print_help(sys.stderr)
-            _LOGGER.warning(
-                f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
-            )
-            sys.exit(1)
-    else:
+                setattr(subcommand_args, looper_config_key, looper_config_item)
+
+    except OSError:
+        parser.print_help(sys.stderr)
         _LOGGER.warning(
-            "This PEP configures looper through the project config. This approach is deprecated and will "
-            "be removed in future versions. Please use a looper config file. For more information see "
-            "looper.databio.org/en/latest/looper-config"
+            f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
         )
+        sys.exit(1)
 
     subcommand_args = enrich_args_via_cfg(
         subcommand_name,

From 7a35c499933078c0d204bfeb8c07a2c132cd8f33 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 25 Jun 2024 15:23:44 -0400
Subject: [PATCH 02/56] re-commit no longer checking for
 subcommand_args.config_file

---
 looper/cli_pydantic.py | 45 +++++++++++++++++-------------------------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 16bb6206..f4fb26fe 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -138,36 +138,27 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
-    if subcommand_args.config_file is None:
-        looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
-        try:
-            if subcommand_args.looper_config:
-                looper_config_dict = read_looper_config_file(
-                    subcommand_args.looper_config
-                )
+    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+    try:
+        if subcommand_args.looper_config:
+            looper_config_dict = read_looper_config_file(subcommand_args.looper_config)
+        else:
+            looper_config_dict = read_looper_dotfile()
+            _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+        cli_modifiers_dict = None
+        for looper_config_key, looper_config_item in looper_config_dict.items():
+            if looper_config_key == CLI_KEY:
+                cli_modifiers_dict = looper_config_item
             else:
-                looper_config_dict = read_looper_dotfile()
-                _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
-
-            cli_modifiers_dict = None
-            for looper_config_key, looper_config_item in looper_config_dict.items():
-                if looper_config_key == CLI_KEY:
-                    cli_modifiers_dict = looper_config_item
-                else:
-                    setattr(subcommand_args, looper_config_key, looper_config_item)
-
-        except OSError:
-            parser.print_help(sys.stderr)
-            _LOGGER.warning(
-                f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
-            )
-            sys.exit(1)
-    else:
+                setattr(subcommand_args, looper_config_key, looper_config_item)
+
+    except OSError:
+        parser.print_help(sys.stderr)
         _LOGGER.warning(
-            "This PEP configures looper through the project config. This approach is deprecated and will "
-            "be removed in future versions. Please use a looper config file. For more information see "
-            "looper.databio.org/en/latest/looper-config"
+            f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
         )
+        sys.exit(1)
 
     subcommand_args = enrich_args_via_cfg(
         subcommand_name,

From 9c3d9e33e93a6e6d7744bf8f1f498010279a3013 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 25 Jun 2024 15:34:13 -0400
Subject: [PATCH 03/56] refactor --looper-config to --config

---
 looper/cli_pydantic.py             |  4 ++--
 looper/command_models/arguments.py |  6 +++---
 looper/command_models/commands.py  |  2 +-
 tests/conftest.py                  |  2 +-
 tests/smoketests/test_other.py     | 34 +++++++++++++++---------------
 tests/smoketests/test_run.py       | 14 ++++++------
 tests/test_comprehensive.py        | 18 ++++++++--------
 7 files changed, 39 insertions(+), 41 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index f4fb26fe..9943f4ca 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -140,8 +140,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
     looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
     try:
-        if subcommand_args.looper_config:
-            looper_config_dict = read_looper_config_file(subcommand_args.looper_config)
+        if subcommand_args.config:
+            looper_config_dict = read_looper_config_file(subcommand_args.config)
         else:
             looper_config_dict = read_looper_dotfile()
             _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 8c484d33..1f86a6d4 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -167,8 +167,9 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Project configuration file",
     )
-    LOOPER_CONFIG = Argument(
-        name="looper_config",
+    CONFIG = Argument(
+        name="config",
+        alias="-c",
         default=(str, None),
         description="Looper configuration file (YAML)",
     )
@@ -237,7 +238,6 @@ class ArgumentEnum(enum.Enum):
     )
     COMPUTE = Argument(
         name="compute",
-        alias="-c",
         default=(List, []),
         description="List of key-value pairs (k1=v1)",
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index e764c99d..9a141454 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -54,7 +54,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     ArgumentEnum.PEP_CONFIG.value,
     ArgumentEnum.OUTPUT_DIR.value,
     ArgumentEnum.CONFIG_FILE.value,
-    ArgumentEnum.LOOPER_CONFIG.value,
+    ArgumentEnum.CONFIG.value,
     ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
     ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
     ArgumentEnum.PIPESTAT.value,
diff --git a/tests/conftest.py b/tests/conftest.py
index ef2176fe..cfc10457 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -141,7 +141,7 @@ def test_args_expansion(pth=None, cmd=None, appendix=list(), dry=True) -> List[s
     if cmd:
         x.append(cmd)
     if pth:
-        x.append("--looper-config")
+        x.append("--config")
         x.append(pth)
     if dry:
         x.append("--dry-run")
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index b90e9b61..9713d16a 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -64,7 +64,7 @@ class TestLooperPipestat:
     def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         "report, table, and check should fail if pipestat is NOT configured."
         tp = prep_temp_pep
-        x = [cmd, "--looper-config", tp]
+        x = [cmd, "--config", tp]
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
 
@@ -73,10 +73,10 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
         tp = prep_temp_pep_pipestat
 
         if cmd in ["run", "runp"]:
-            x = [cmd, "--looper-config", tp, "--dry-run"]
+            x = [cmd, "--config", tp, "--dry-run"]
         else:
             # Not every command supports dry run
-            x = [cmd, "--looper-config", tp]
+            x = [cmd, "--config", tp]
 
         try:
             result = main(test_args=x)
@@ -120,7 +120,7 @@ def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags):
         with open(pipestat_project_file, "w") as f:
             dump(pipestat_project_data, f)
 
-        x = ["rerun", "--looper-config", tp]
+        x = ["rerun", "--config", tp]
         try:
             result = main(test_args=x)
         except Exception:
@@ -137,7 +137,7 @@ def test_rerun_no_pipestat(self, prep_temp_pep, pipeline_name, flags):
         tp = prep_temp_pep
         _make_flags(tp, flags, pipeline_name)
 
-        x = ["rerun", "--looper-config", tp]
+        x = ["rerun", "--config", tp]
         try:
             result = main(test_args=x)
         except Exception:
@@ -157,7 +157,7 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags_pipestat(tp, flag_id, pipeline_name)
 
-        x = ["check", "--looper-config", tp]
+        x = ["check", "--config", tp]
 
         try:
             results = main(test_args=x)
@@ -176,7 +176,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         _make_flags_pipestat(tp, flag_id, pipeline_name)
         _make_flags_pipestat(tp, FLAGS[1], pipeline_name)
 
-        x = ["check", "--looper-config", tp]
+        x = ["check", "--config", tp]
         # Multiple flag files SHOULD cause pipestat to throw an assertion error
         if flag_id != FLAGS[1]:
             with pytest.raises(AssertionError):
@@ -189,7 +189,7 @@ def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags_pipestat(tp, flag_id, pipeline_name)
 
-        x = ["check", "--looper-config", tp]
+        x = ["check", "--config", tp]
         try:
             results = main(test_args=x)
             result_key = list(results.keys())[0]
@@ -232,7 +232,7 @@ def test_selecting_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "--looper-config", tp, "--sel-flag", "completed", "--dry-run"]
+        x = ["run", "--config", tp, "--sel-flag", "completed", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -273,7 +273,7 @@ def test_excluding_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "--looper-config", tp, "--exc-flag", "running", "--dry-run"]
+        x = ["run", "--config", tp, "--exc-flag", "running", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -318,7 +318,7 @@ def test_excluding_multi_flags_works(
 
         x = [
             "run",
-            "--looper-config",
+            "--config",
             tp,
             "--exc-flag",
             "completed",
@@ -368,7 +368,7 @@ def test_selecting_multi_flags_works(
         x = [
             "run",
             "--dry-run",
-            "--looper-config",
+            "--config",
             tp,
             "--sel-flag",
             "completed",
@@ -419,7 +419,7 @@ def test_selecting_attr_and_flags_works(
         x = [
             "run",
             "--dry-run",
-            "--looper-config",
+            "--config",
             tp,
             "--sel-flag",
             "completed",
@@ -472,7 +472,7 @@ def test_excluding_attr_and_flags_works(
         x = [
             "run",
             "--dry-run",
-            "--looper-config",
+            "--config",
             tp,
             "--exc-flag",
             "completed",
@@ -534,7 +534,7 @@ def test_excluding_toggle_attr(
         x = [
             "run",
             "--dry-run",
-            "--looper-config",
+            "--config",
             tp,
             "--sel-attr",
             "toggle",
@@ -596,7 +596,7 @@ def test_including_toggle_attr(
         x = [
             "run",
             "--dry-run",
-            "--looper-config",
+            "--config",
             tp,
             "--sel-attr",
             "toggle",
@@ -620,7 +620,7 @@ class TestLooperInspect:
     def test_inspect_config(self, prep_temp_pep, cmd):
         "Checks inspect command"
         tp = prep_temp_pep
-        x = [cmd, "--looper-config", tp]
+        x = [cmd, "--config", tp]
         try:
             results = main(test_args=x)
         except Exception:
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 6c0bdd8d..4a35f4f4 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -16,7 +16,7 @@
 def test_cli(prep_temp_pep):
     tp = prep_temp_pep
 
-    x = ["run", "--looper-config", tp, "--dry-run"]
+    x = ["run", "--config", tp, "--dry-run"]
     try:
         main(test_args=x)
     except Exception:
@@ -26,20 +26,20 @@ def test_cli(prep_temp_pep):
 def test_cli_shortform(prep_temp_pep):
     tp = prep_temp_pep
 
-    x = ["run", "--looper-config", tp, "-d"]
+    x = ["run", "--config", tp, "-d"]
     try:
         main(test_args=x)
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    x = ["run", "--looper-config", tp, "-d", "-l", "2"]
+    x = ["run", "--config", tp, "-d", "-l", "2"]
     try:
         main(test_args=x)
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
     tp = prep_temp_pep
-    x = ["run", "--looper-config", tp, "-d", "-n", "2"]
+    x = ["run", "--config", tp, "-d", "-n", "2"]
     try:
         main(test_args=x)
     except Exception:
@@ -49,7 +49,7 @@ def test_cli_shortform(prep_temp_pep):
 def test_running_csv_pep(prep_temp_pep_csv):
     tp = prep_temp_pep_csv
 
-    x = ["run", "--looper-config", tp, "--dry-run"]
+    x = ["run", "--config", tp, "--dry-run"]
     try:
         main(test_args=x)
     except Exception:
@@ -83,9 +83,7 @@ class TestLooperBothRuns:
     def test_looper_cfg_invalid(self, cmd):
         """Verify looper does not accept invalid cfg paths"""
 
-        x = test_args_expansion(
-            cmd, "--looper-config", "jdfskfds/dsjfklds/dsjklsf.yaml"
-        )
+        x = test_args_expansion(cmd, "--config", "jdfskfds/dsjfklds/dsjklsf.yaml")
         with pytest.raises(SystemExit):
             result = main(test_args=x)
             print(result)
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 9b857f8f..f3aadbce 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -23,7 +23,7 @@ def test_comprehensive_advanced_looper_no_pipestat(prep_temp_pep):
 
     path_to_looper_config = prep_temp_pep
 
-    x = ["run", "--looper-config", path_to_looper_config]
+    x = ["run", "--config", path_to_looper_config]
 
     try:
         results = main(test_args=x)
@@ -48,7 +48,7 @@ def test_comprehensive_looper_no_pipestat(prep_temp_pep_basic):
     with open(basic_project_file, "w") as f:
         dump(basic_project_data, f)
 
-    x = ["run", "--looper-config", path_to_looper_config]
+    x = ["run", "--config", path_to_looper_config]
 
     try:
         results = main(test_args=x)
@@ -85,7 +85,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     with open(pipestat_project_file, "w") as f:
         dump(pipestat_project_data, f)
 
-    x = [cmd, "--looper-config", path_to_looper_config]
+    x = [cmd, "--config", path_to_looper_config]
 
     try:
         result = main(test_args=x)
@@ -110,7 +110,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     psm.set_status(record_identifier="frog_2", status_identifier="completed")
 
     # Now use looper check to get statuses
-    x = ["check", "--looper-config", path_to_looper_config]
+    x = ["check", "--config", path_to_looper_config]
 
     try:
         result = main(test_args=x)
@@ -119,7 +119,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
     # Now use looper check to get project level statuses
-    x = ["check", "--looper-config", path_to_looper_config, "--project"]
+    x = ["check", "--config", path_to_looper_config, "--project"]
 
     try:
         result = main(test_args=x)
@@ -130,7 +130,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     # TEST LOOPER REPORT
 
-    x = ["report", "--looper-config", path_to_looper_config]
+    x = ["report", "--config", path_to_looper_config]
 
     try:
         result = main(test_args=x)
@@ -140,7 +140,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     # TEST LOOPER Table
 
-    x = ["table", "--looper-config", path_to_looper_config]
+    x = ["table", "--config", path_to_looper_config]
 
     try:
         result = main(test_args=x)
@@ -153,7 +153,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     x = [
         "destroy",
-        "--looper-config",
+        "--config",
         path_to_looper_config,
         "--force-yes",
     ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
@@ -178,7 +178,7 @@ def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
     # TODO need to add way to check if user is logged into pephub and then run test otherwise skip
     path_to_looper_config = prep_temp_pep_pephub
 
-    x = ["run", "--looper-config", path_to_looper_config]
+    x = ["run", "--config", path_to_looper_config]
 
     try:
         results = main(test_args=x)

From d8437aa7f9e0ee09c7fa39d6c177ed510d01a970 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 25 Jun 2024 16:19:38 -0400
Subject: [PATCH 04/56] remove --config-file in favor of using --pep-config

---
 looper/cli_pydantic.py             | 10 +++++-----
 looper/command_models/arguments.py |  5 -----
 looper/command_models/commands.py  |  1 -
 looper/utils.py                    | 11 ++---------
 4 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 9943f4ca..8b114a82 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -182,12 +182,12 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         subcommand_args.ignore_flags = True
 
     # Initialize project
-    if is_PEP_file_type(subcommand_args.config_file) and os.path.exists(
-        subcommand_args.config_file
+    if is_PEP_file_type(subcommand_args.pep_config) and os.path.exists(
+        subcommand_args.pep_config
     ):
         try:
             p = Project(
-                cfg=subcommand_args.config_file,
+                cfg=subcommand_args.pep_config,
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
@@ -200,14 +200,14 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         except yaml.parser.ParserError as e:
             _LOGGER.error(f"Project config parse failed -- {e}")
             sys.exit(1)
-    elif is_pephub_registry_path(subcommand_args.config_file):
+    elif is_pephub_registry_path(subcommand_args.pep_config):
         if vars(subcommand_args)[SAMPLE_PL_ARG]:
             p = Project(
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
                 project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=subcommand_args.config_file
+                    registry_path=subcommand_args.pep_config
                 ),
                 **{
                     attr: getattr(subcommand_args, attr)
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 1f86a6d4..9d495393 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -162,11 +162,6 @@ class ArgumentEnum(enum.Enum):
         default=(int, None),
         description="Skip samples by numerical index",
     )
-    CONFIG_FILE = Argument(
-        name="config_file",
-        default=(str, None),
-        description="Project configuration file",
-    )
     CONFIG = Argument(
         name="config",
         alias="-c",
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 9a141454..8ae037e0 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -53,7 +53,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     ArgumentEnum.SKIP.value,
     ArgumentEnum.PEP_CONFIG.value,
     ArgumentEnum.OUTPUT_DIR.value,
-    ArgumentEnum.CONFIG_FILE.value,
     ArgumentEnum.CONFIG.value,
     ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
     ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
diff --git a/looper/utils.py b/looper/utils.py
index 4eb8ca4b..ebcd803e 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -273,7 +273,7 @@ def enrich_args_via_cfg(
     """
     cfg_args_all = (
         _get_subcommand_args(subcommand_name, parser_args)
-        if os.path.exists(parser_args.config_file)
+        if os.path.exists(parser_args.pep_config)
         else dict()
     )
 
@@ -360,7 +360,7 @@ def _get_subcommand_args(subcommand_name, parser_args):
     """
     args = dict()
     cfg = peppyProject(
-        parser_args.config_file,
+        parser_args.pep_config,
         defer_samples_creation=True,
         amendments=parser_args.amend,
     )
@@ -579,15 +579,8 @@ def read_looper_config_file(looper_config_path: str) -> dict:
         dp_data = yaml.safe_load(dotfile)
 
     if PEP_CONFIG_KEY in dp_data:
-        # Looper expects the config path to live at looper.config_file
-        # However, user may wish to access the pep at looper.pep_config
-        return_dict[PEP_CONFIG_FILE_KEY] = dp_data[PEP_CONFIG_KEY]
         return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
 
-    # TODO: delete it in looper 2.0
-    elif DOTFILE_CFG_PTH_KEY in dp_data:
-        return_dict[PEP_CONFIG_FILE_KEY] = dp_data[DOTFILE_CFG_PTH_KEY]
-
     else:
         raise MisconfigurationException(
             f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key"

From a62ff3545c46e30c0926e235ab5a3ac4f59c6222 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 27 Jun 2024 16:13:15 -0400
Subject: [PATCH 05/56] work towards using consolidated piface, update generic
 schema

---
 looper/conductor.py                           |  9 ++-
 looper/pipeline_interface.py                  |  9 ---
 .../pipeline_interface_schema_generic.yaml    | 20 ++++--
 looper/utils.py                               | 65 ++++++++++---------
 4 files changed, 56 insertions(+), 47 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 93598431..70960d1f 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -198,6 +198,9 @@ def __init__(
 
         self.collate = collate
         self.section_key = PROJECT_PL_KEY if self.collate else SAMPLE_PL_KEY
+        self.pipeline_interface_type = (
+            "project_interface" if self.collate else "sample_interface"
+        )
         self.pl_iface = pipeline_interface
         self.pl_name = self.pl_iface.pipeline_name
         self.prj = prj
@@ -681,7 +684,11 @@ def write_script(self, pool, size):
             pipeline=self.pl_iface,
             compute=self.prj.dcc.compute,
         )
-        templ = self.pl_iface["command_template"]
+
+        if self.pipeline_interface_type is None:
+            templ = self.pl_iface["command_template"]
+        else:
+            templ = self.pl_iface[self.pipeline_interface_type]["command_template"]
         if not self.override_extra:
             extras_template = (
                 EXTRA_PROJECT_CMD_TEMPLATE
diff --git a/looper/pipeline_interface.py b/looper/pipeline_interface.py
index f796354a..387c6d49 100644
--- a/looper/pipeline_interface.py
+++ b/looper/pipeline_interface.py
@@ -56,15 +56,6 @@ def __init__(self, config, pipeline_type=None):
             )
         self.update(config)
         self._validate(schema_src=PIFACE_SCHEMA_SRC)
-        if "path" in self:
-            warn(
-                message="'path' specification as a top-level pipeline "
-                "interface key is deprecated and will be removed with "
-                "the next release. Please use 'paths' section "
-                "from now on.",
-                category=DeprecationWarning,
-            )
-            self._expand_paths(["path"])
         self._expand_paths(["compute", "dynamic_variables_script_path"])
 
     @property
diff --git a/looper/schemas/pipeline_interface_schema_generic.yaml b/looper/schemas/pipeline_interface_schema_generic.yaml
index 3d1b4ea1..8528b613 100644
--- a/looper/schemas/pipeline_interface_schema_generic.yaml
+++ b/looper/schemas/pipeline_interface_schema_generic.yaml
@@ -9,12 +9,20 @@ properties:
     type: string
     enum: ["project", "sample"]
     description: "type of the pipeline, either 'project' or 'sample'"
-  command_template:
-    type: string
-    description: "Jinja2-like template to construct the command to run"
-  path:
-    type: string
-    description: "path to the pipeline program. Relative to pipeline interface file or absolute."
+  sample_interface:
+    type: object
+    description: "Section that defines compute environment settings"
+    properties:
+      command_template:
+        type: string
+        description: "Jinja2-like template to construct the command to run"
+  project_interface:
+    type: object
+    description: "Section that defines compute environment settings"
+    properties:
+      command_template:
+        type: string
+        description: "Jinja2-like template to construct the command to run"
   compute:
     type: object
     description: "Section that defines compute environment settings"
diff --git a/looper/utils.py b/looper/utils.py
index ebcd803e..9c66d33d 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -20,7 +20,7 @@
 
 from .const import *
 from .command_models.commands import SUPPORTED_COMMANDS
-from .exceptions import MisconfigurationException
+from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
 
 _LOGGER = getLogger(__name__)
 
@@ -538,7 +538,7 @@ def initiate_looper_config(
 
 def determine_pipeline_type(piface_path: str, looper_config_path: str):
     """
-    Read pipeline interface from disk and determine if pipeline type is sample or project-level
+    Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both
 
 
     :param str piface_path: path to pipeline_interface
@@ -558,9 +558,17 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
     except FileNotFoundError:
         return None, None
 
-    pipeline_type = piface_dict.get("pipeline_type", None)
+    pipeline_types = []
+    if piface_dict.get("sample_interface", None):
+        pipeline_types.append(PipelineLevel.SAMPLE.value)
+    if piface_dict.get("project_interface", None):
+        pipeline_types.append(PipelineLevel.PROJECT.value)
 
-    return pipeline_type, piface_path
+    if pipeline_types == []:
+        # TODO WARN USER THEY MUST GIVE EITHER A SAMPLE OR PROJECT INTERFACE
+        return None, None
+
+    return pipeline_types, piface_path
 
 
 def read_looper_config_file(looper_config_path: str) -> dict:
@@ -606,36 +614,31 @@ def read_looper_config_file(looper_config_path: str) -> dict:
 
         dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
 
-        if isinstance(dp_data.get(PIPELINE_INTERFACES_KEY), dict) and (
-            dp_data.get(PIPELINE_INTERFACES_KEY).get("sample")
-            or dp_data.get(PIPELINE_INTERFACES_KEY).get("project")
-        ):
-            # Support original nesting of pipeline interfaces under "sample" and "project"
-            return_dict[SAMPLE_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get(
-                "sample"
-            )
-            return_dict[PROJECT_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get(
-                "project"
+        all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY)
+        sample_pifaces = []
+        project_pifaces = []
+        if isinstance(all_pipeline_interfaces, str):
+            all_pipeline_interfaces = [all_pipeline_interfaces]
+        for piface in all_pipeline_interfaces:
+            pipeline_types, piface_path = determine_pipeline_type(
+                piface, looper_config_path
             )
-        else:
-            # infer pipeline type based from interface instead of nested keys: https://github.com/pepkit/looper/issues/465
-            all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY)
-            sample_pifaces = []
-            project_pifaces = []
-            if isinstance(all_pipeline_interfaces, str):
-                all_pipeline_interfaces = [all_pipeline_interfaces]
-            for piface in all_pipeline_interfaces:
-                pipeline_type, piface_path = determine_pipeline_type(
-                    piface, looper_config_path
-                )
-                if pipeline_type == PipelineLevel.SAMPLE.value:
+            # if pipeline_types is None:
+            #     raise PipelineInterfaceConfigError(
+            #         f"'sample_interface and/or project_interface must be defined in each pipeline interface."
+            #     )
+            # This will append the same, consolidated piface to two different lists
+            # In reality only the command templates are the differentiator
+            if pipeline_types is not None:
+                # TODO should we raise an exception here? I guess you can amend samples with interfaces...
+                if PipelineLevel.SAMPLE.value in pipeline_types:
                     sample_pifaces.append(piface_path)
-                elif pipeline_type == PipelineLevel.PROJECT.value:
+                if PipelineLevel.PROJECT.value in pipeline_types:
                     project_pifaces.append(piface_path)
-            if len(sample_pifaces) > 0:
-                return_dict[SAMPLE_PL_ARG] = sample_pifaces
-            if len(project_pifaces) > 0:
-                return_dict[PROJECT_PL_ARG] = project_pifaces
+        if len(sample_pifaces) > 0:
+            return_dict[SAMPLE_PL_ARG] = sample_pifaces
+        if len(project_pifaces) > 0:
+            return_dict[PROJECT_PL_ARG] = project_pifaces
 
     else:
         _LOGGER.warning(

From 168f012d61b757c1ce86148f85f3a4b668877828 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 27 Jun 2024 16:52:27 -0400
Subject: [PATCH 06/56] pull updated hell_looper dev for #493

---
 tests/data/hello_looper-dev/advanced/.looper.yaml      | 10 ++++------
 .../advanced/.looper_advanced_pipestat.yaml            |  5 ++---
 .../advanced/pipeline/pipeline_interface1_project.yaml |  6 +++---
 .../advanced/pipeline/pipeline_interface1_sample.yaml  |  6 +++---
 .../advanced/pipeline/pipeline_interface2_project.yaml |  6 +++---
 .../advanced/pipeline/pipeline_interface2_sample.yaml  |  6 +++---
 .../pipeline/pipestat_pipeline_interface1_sample.yaml  |  6 +++---
 .../pipeline/pipestat_pipeline_interface2_sample.yaml  |  6 +++---
 tests/data/hello_looper-dev/csv/.looper.yaml           |  2 +-
 .../csv/pipeline/pipeline_interface.yaml               |  8 ++++++--
 .../csv/pipeline/pipeline_interface_project.yaml       |  6 ------
 tests/data/hello_looper-dev/intermediate/.looper.yaml  |  2 +-
 .../hello_looper-dev/intermediate/.looper_project.yaml |  4 ----
 .../intermediate/pipeline/pipeline_interface.yaml      |  9 ++++++---
 .../pipeline/pipeline_interface_project.yaml           |  6 ------
 tests/data/hello_looper-dev/minimal/.looper.yaml       |  2 +-
 .../minimal/pipeline/pipeline_interface.yaml           |  6 +++---
 tests/data/hello_looper-dev/pephub/.looper.yaml        |  2 +-
 .../pephub/pipeline/pipeline_interface.yaml            |  8 ++++++--
 .../pephub/pipeline/pipeline_interface_project.yaml    |  6 ------
 tests/data/hello_looper-dev/pipestat/.looper.yaml      |  3 +--
 .../pipestat/.looper_pipestat_shell.yaml               |  2 +-
 .../pipestat/looper_pipestat_config.yaml               |  7 -------
 .../pipestat/pipeline_pipestat/pipeline_interface.yaml |  9 ++++++---
 .../pipeline_pipestat/pipeline_interface_project.yaml  |  8 --------
 .../pipeline_pipestat/pipeline_interface_shell.yaml    |  6 +++---
 26 files changed, 60 insertions(+), 87 deletions(-)
 delete mode 100644 tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml
 delete mode 100644 tests/data/hello_looper-dev/intermediate/.looper_project.yaml
 delete mode 100644 tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml
 delete mode 100644 tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml
 delete mode 100644 tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml
 delete mode 100644 tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml

diff --git a/tests/data/hello_looper-dev/advanced/.looper.yaml b/tests/data/hello_looper-dev/advanced/.looper.yaml
index d2c5797f..26942bd9 100644
--- a/tests/data/hello_looper-dev/advanced/.looper.yaml
+++ b/tests/data/hello_looper-dev/advanced/.looper.yaml
@@ -1,10 +1,8 @@
 pep_config: project/project_config.yaml
 output_dir: "results"
 pipeline_interfaces:
-  sample:
-    - ../pipeline/pipeline_interface1_sample.yaml
-    - ../pipeline/pipeline_interface2_sample.yaml
-  project:
-    - ../pipeline/pipeline_interface1_project.yaml
-    - ../pipeline/pipeline_interface2_project.yaml
+    - pipeline/pipeline_interface1_sample.yaml
+    - pipeline/pipeline_interface2_sample.yaml
+    - pipeline/pipeline_interface1_project.yaml
+    - pipeline/pipeline_interface2_project.yaml
 
diff --git a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
index 74da1a3f..3c4963c2 100644
--- a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
+++ b/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
@@ -1,9 +1,8 @@
 pep_config: project/project_config.yaml
 output_dir: "results"
 pipeline_interfaces:
-  sample:
-    - ../pipeline/pipestat_pipeline_interface1_sample.yaml
-    - ../pipeline/pipestat_pipeline_interface2_sample.yaml
+    - pipeline/pipestat_pipeline_interface1_sample.yaml
+    - pipeline/pipestat_pipeline_interface2_sample.yaml
 pipestat:
   results_file_path: results.yaml
   flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
index 2a23d321..534905ca 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
@@ -1,9 +1,9 @@
 pipeline_name: PIPELINE1
-pipeline_type: project
 output_schema: output_schema.yaml
 var_templates:
   path: "{looper.piface_dir}/col_pipeline1.py"
-command_template: >
-  python3 {pipeline.var_templates.path} --project-name {project.name}
+project_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --project-name {project.name}
 
 
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
index 8e79b7ae..d0d60849 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
@@ -1,5 +1,4 @@
 pipeline_name: PIPELINE1
-pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: output_schema.yaml
 var_templates:
@@ -7,7 +6,8 @@ var_templates:
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
-command_template: >
-  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+sample_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 
 
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
index 824b7e09..df557d82 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
@@ -1,10 +1,10 @@
 pipeline_name: OTHER_PIPELINE2
-pipeline_type: project
 output_schema: output_schema.yaml
 var_templates:
   path: "{looper.piface_dir}/col_pipeline2.py"
-command_template: >
-  python3 {pipeline.var_templates.path} --project-name {project.name}
+project_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --project-name {project.name}
 compute:
   size_dependent_variables: resources-project.tsv
 
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
index 589aef6d..0329d33a 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
@@ -1,13 +1,13 @@
 pipeline_name: OTHER_PIPELINE2
-pipeline_type: sample
 output_schema: output_schema.yaml
 var_templates:
   path: "{looper.piface_dir}/other_pipeline2.py"
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
-command_template: >
-  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+sample_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 compute:
   size_dependent_variables: resources-sample.tsv
 
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
index e687ea0d..4bdbab1f 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
@@ -1,5 +1,4 @@
 pipeline_name: example_pipestat_pipeline
-pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: pipestat_output_schema.yaml
 var_templates:
@@ -7,7 +6,8 @@ var_templates:
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
-command_template: >
-  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+sample_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 
 
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
index bac3ea3d..3fa6829c 100644
--- a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
+++ b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
@@ -1,5 +1,4 @@
 pipeline_name: example_pipestat_pipeline
-pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: pipestat_output_schema.yaml
 var_templates:
@@ -7,8 +6,9 @@ var_templates:
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
-command_template: >
-  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+sample_interface:
+  command_template: >
+    python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 compute:
   size_dependent_variables: resources-sample.tsv
 
diff --git a/tests/data/hello_looper-dev/csv/.looper.yaml b/tests/data/hello_looper-dev/csv/.looper.yaml
index c88f0c9a..886b301c 100644
--- a/tests/data/hello_looper-dev/csv/.looper.yaml
+++ b/tests/data/hello_looper-dev/csv/.looper.yaml
@@ -2,4 +2,4 @@ pep_config: project/sample_annotation.csv # local path to CSV
 # pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
 output_dir: "results"
 pipeline_interfaces:
-  sample: pipeline/pipeline_interface.yaml
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
index 732e6976..1b8a77eb 100644
--- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
@@ -2,5 +2,9 @@ pipeline_name: count_lines
 pipeline_type: sample
 var_templates:
   pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} {sample.file}
+sample_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} {sample.file}
+project_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} "data/*.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml
deleted file mode 100644
index 9063c7d6..00000000
--- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pipeline_name: count_lines
-pipeline_type: project
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev/intermediate/.looper.yaml b/tests/data/hello_looper-dev/intermediate/.looper.yaml
index 19fac81d..4fcf5672 100644
--- a/tests/data/hello_looper-dev/intermediate/.looper.yaml
+++ b/tests/data/hello_looper-dev/intermediate/.looper.yaml
@@ -2,4 +2,4 @@ pep_config: project/project_config.yaml # local path to pep config
 # pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
 output_dir: "results"
 pipeline_interfaces:
-  sample: pipeline/pipeline_interface.yaml
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/.looper_project.yaml b/tests/data/hello_looper-dev/intermediate/.looper_project.yaml
deleted file mode 100644
index b44ef03b..00000000
--- a/tests/data/hello_looper-dev/intermediate/.looper_project.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-pep_config: project/project_config.yaml # local path to pep config
-output_dir: "results"
-pipeline_interfaces:
-  project: pipeline/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
index 732e6976..dde7c393 100644
--- a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
@@ -1,6 +1,9 @@
 pipeline_name: count_lines
-pipeline_type: sample
 var_templates:
   pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} {sample.file}
+sample_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} {sample.file}
+project_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} "data/*.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml
deleted file mode 100644
index 9063c7d6..00000000
--- a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pipeline_name: count_lines
-pipeline_type: project
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev/minimal/.looper.yaml b/tests/data/hello_looper-dev/minimal/.looper.yaml
index 19fac81d..4fcf5672 100644
--- a/tests/data/hello_looper-dev/minimal/.looper.yaml
+++ b/tests/data/hello_looper-dev/minimal/.looper.yaml
@@ -2,4 +2,4 @@ pep_config: project/project_config.yaml # local path to pep config
 # pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
 output_dir: "results"
 pipeline_interfaces:
-  sample: pipeline/pipeline_interface.yaml
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
index 732e6976..58ddbf2b 100644
--- a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
@@ -1,6 +1,6 @@
 pipeline_name: count_lines
-pipeline_type: sample
 var_templates:
   pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} {sample.file}
+sample_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev/pephub/.looper.yaml b/tests/data/hello_looper-dev/pephub/.looper.yaml
index 00e60ded..c2d74e0c 100644
--- a/tests/data/hello_looper-dev/pephub/.looper.yaml
+++ b/tests/data/hello_looper-dev/pephub/.looper.yaml
@@ -1,4 +1,4 @@
 pep_config: pepkit/hello_looper:default # pephub registry path or local path
 output_dir: results
 pipeline_interfaces:
-  sample: pipeline/pipeline_interface.yaml
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
index 732e6976..9bea7b43 100644
--- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
@@ -2,5 +2,9 @@ pipeline_name: count_lines
 pipeline_type: sample
 var_templates:
   pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} {sample.file}
+sample_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} {sample.file}
+project_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml
deleted file mode 100644
index 9063c7d6..00000000
--- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pipeline_name: count_lines
-pipeline_type: project
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev/pipestat/.looper.yaml b/tests/data/hello_looper-dev/pipestat/.looper.yaml
index 852c6fa4..3fa1947c 100644
--- a/tests/data/hello_looper-dev/pipestat/.looper.yaml
+++ b/tests/data/hello_looper-dev/pipestat/.looper.yaml
@@ -1,8 +1,7 @@
 pep_config: ./project/project_config.yaml # pephub registry path or local path
 output_dir: ./results
 pipeline_interfaces:
-  sample:  ./pipeline_pipestat/pipeline_interface.yaml
-  project: ./pipeline_pipestat/pipeline_interface_project.yaml
+  - pipeline_pipestat/pipeline_interface.yaml
 pipestat:
   results_file_path: results.yaml
   flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
index fb645a9b..6ecbdfc1 100644
--- a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
+++ b/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
@@ -1,7 +1,7 @@
 pep_config: ./project/project_config.yaml # pephub registry path or local path
 output_dir: ./results
 pipeline_interfaces:
-  sample:  ./pipeline_pipestat/pipeline_interface_shell.yaml
+  - pipeline_pipestat/pipeline_interface_shell.yaml
 pipestat:
   results_file_path: results.yaml
   flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml b/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml
deleted file mode 100644
index 0a04ac6f..00000000
--- a/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-results_file_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results.yaml
-flag_file_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results/flags
-output_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results
-record_identifier: frog_2
-schema_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./pipeline_pipestat/pipestat_output_schema.yaml
-pipeline_name: test_pipe
-pipeline_type: sample
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
index e5a14402..ee9a8d5c 100644
--- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
@@ -1,5 +1,8 @@
 pipeline_name: example_pipestat_pipeline
-pipeline_type: sample
 output_schema: pipestat_output_schema.yaml
-command_template: >
-  python3 {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema}
\ No newline at end of file
+sample_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema}
+project_interface:
+  command_template: >
+    {pipeline.var_templates.pipeline} "data/*.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
deleted file mode 100644
index 2237c2f3..00000000
--- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-pipeline_name: example_pipestat_project_pipeline
-pipeline_type: project
-output_schema: pipestat_output_schema.yaml
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-command_template: >
-  {pipeline.var_templates.pipeline} "data/*.txt"
-
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
index 82df8b94..2fed0285 100644
--- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
+++ b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
@@ -1,5 +1,5 @@
 pipeline_name: example_pipestat_pipeline
-pipeline_type: sample
 output_schema: pipestat_output_schema.yaml
-command_template: >
-  {looper.piface_dir}/count_lines_pipestat.sh {sample.file} {sample.sample_name} {pipestat.config_file}
\ No newline at end of file
+sample_interface:
+  command_template: >
+    {looper.piface_dir}/count_lines_pipestat.sh {sample.file} {sample.sample_name} {pipestat.config_file}
\ No newline at end of file

From 2d2475b90e9dad77cf55ccad18925b9e7fb90840 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 27 Jun 2024 17:43:26 -0400
Subject: [PATCH 07/56] fix pipeline_type for pipestat usage with consolidated
 pipelines, fix remaning tests

---
 looper/cli_pydantic.py       |  2 +-
 looper/project.py            | 17 ++++++++++-------
 tests/smoketests/test_run.py | 10 ++++------
 tests/test_comprehensive.py  |  2 +-
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 8b114a82..954bdef6 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -243,7 +243,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
         is_pipestat_configured = (
             prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
-            if getattr(subcommand_args, "project", None)
+            if getattr(subcommand_args, "project", None) or subcommand_name == "runp"
             else prj._check_if_pipestat_configured()
         )
 
diff --git a/looper/project.py b/looper/project.py
index 7a652db7..e485bfa5 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -413,10 +413,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value):
                 pipestat_config_path = self._check_for_existing_pipestat_config(piface)
 
                 if not pipestat_config_path:
-                    self._create_pipestat_config(piface)
+                    self._create_pipestat_config(piface, pipeline_type)
                 else:
                     piface.psm = PipestatManager(
-                        config_file=pipestat_config_path, multi_pipelines=True
+                        config_file=pipestat_config_path,
+                        multi_pipelines=True,
+                        pipeline_type="sample",
                     )
 
         elif pipeline_type == PipelineLevel.PROJECT.value:
@@ -426,10 +428,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value):
                 )
 
                 if not pipestat_config_path:
-                    self._create_pipestat_config(prj_piface)
+                    self._create_pipestat_config(prj_piface, pipeline_type)
                 else:
                     prj_piface.psm = PipestatManager(
-                        config_file=pipestat_config_path, multi_pipelines=True
+                        config_file=pipestat_config_path,
+                        multi_pipelines=True,
+                        pipeline_type="project",
                     )
         else:
             _LOGGER.error(
@@ -469,7 +473,7 @@ def _check_for_existing_pipestat_config(self, piface):
         else:
             return None
 
-    def _create_pipestat_config(self, piface):
+    def _create_pipestat_config(self, piface, pipeline_type):
         """
         Each piface needs its own config file and associated psm
         """
@@ -512,11 +516,10 @@ def _create_pipestat_config(self, piface):
             pipestat_config_dict.update({"pipeline_name": piface.data["pipeline_name"]})
         else:
             pipeline_name = None
-        if "pipeline_type" in piface.data:
-            pipestat_config_dict.update({"pipeline_type": piface.data["pipeline_type"]})
 
         # Warn user if there is a mismatch in pipeline_names from sources!!!
         if pipeline_name != output_schema_pipeline_name:
+            # TODO Raise exception here.
             _LOGGER.warning(
                 msg=f"Pipeline name mismatch detected. Pipeline interface: {pipeline_name}  Output schema: {output_schema_pipeline_name}  Defaulting to pipeline_interface value."
             )
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 4a35f4f4..d120722e 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -162,8 +162,7 @@ def test_looper_single_pipeline(self, prep_temp_pep):
         with mod_yaml_data(tp) as config_data:
 
             pifaces = config_data[PIPELINE_INTERFACES_KEY]
-            config_data[PIPELINE_INTERFACES_KEY]["sample"] = pifaces["sample"][1]
-            del config_data[PIPELINE_INTERFACES_KEY]["project"]
+            config_data[PIPELINE_INTERFACES_KEY] = pifaces[0]
 
         x = test_args_expansion(tp, "run")
         try:
@@ -193,7 +192,7 @@ def test_looper_cli_pipeline(self, prep_temp_pep):
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
             pifaces = config_data[PIPELINE_INTERFACES_KEY]
-            pi_pth = pifaces["sample"][1]
+            pi_pth = pifaces[1]
         x = test_args_expansion(tp, "run", ["--sample-pipeline-interfaces", pi_pth])
         try:
             result = main(test_args=x)
@@ -224,7 +223,7 @@ def test_looper_pipeline_not_found(self, prep_temp_pep):
         """
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
-            config_data[PIPELINE_INTERFACES_KEY]["sample"] = ["bogus"]
+            config_data[PIPELINE_INTERFACES_KEY] = ["bogus"]
         x = test_args_expansion(tp, "run")
         try:
             result = main(test_args=x)
@@ -344,10 +343,9 @@ def test_looper_single_pipeline(self, prep_temp_pep):
 
         with mod_yaml_data(tp) as config_data:
             # Modifying in this way due to https://github.com/pepkit/looper/issues/474
-            config_data[PIPELINE_INTERFACES_KEY]["project"] = os.path.join(
+            config_data[PIPELINE_INTERFACES_KEY] = os.path.join(
                 os.path.dirname(tp), "pipeline/pipeline_interface1_project.yaml"
             )
-            del config_data[PIPELINE_INTERFACES_KEY]["sample"]
 
         print(tp)
         x = test_args_expansion(tp, "runp")
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index f3aadbce..cb79b356 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -123,7 +123,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     try:
         result = main(test_args=x)
-        assert result == {"example_pipestat_project_pipeline": {"project": "unknown"}}
+        assert result == {"example_pipestat_pipeline": {"project": "unknown"}}
 
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))

From eaab8f9969da099a20474067d0d6d3fada7ed60a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:35:46 -0400
Subject: [PATCH 08/56] refactor looper init tutorial and add some color to
 terminal using rich

---
 looper/cli_pydantic.py       | 35 +++++++++----
 looper/utils.py              | 95 ++++++++++++++++++++++++++++++++++++
 looper_init.py               | 68 --------------------------
 tests/smoketests/test_run.py |  2 +-
 4 files changed, 121 insertions(+), 79 deletions(-)
 delete mode 100644 looper_init.py

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 954bdef6..a9b762c3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -53,10 +53,11 @@
     init_generic_pipeline,
     read_yaml_file,
     inspect_looper_config_file,
-    is_PEP_file_type,
+    is_PEP_file_type, looper_config_tutorial,
 )
 
 from typing import List, Tuple
+from rich.console import Console
 
 
 def opt_attr_pair(name: str) -> Tuple[str, str]:
@@ -122,16 +123,30 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         sys.exit(1)
 
     if subcommand_name == "init":
-        return int(
-            not initiate_looper_config(
-                dotfile_path(),
-                subcommand_args.pep_config,
-                subcommand_args.output_dir,
-                subcommand_args.sample_pipeline_interfaces,
-                subcommand_args.project_pipeline_interfaces,
-                subcommand_args.force_yes,
+
+        console = Console()
+        console.clear()
+        console.rule(f"\n[dark_goldenrod]Looper initialization[/dark_goldenrod]")
+        console.print("[bold]Would you like to follow a guided tutorial?[/bold]  [green]Y[/green] / [red]n[/red]...")
+
+        selection = None
+        while selection not in ['y','Y','n','N']:
+            selection = console.input("\nSelection: ")
+
+        if selection in ['n', 'N']:
+
+            return int(
+                not initiate_looper_config(
+                    dotfile_path(),
+                    subcommand_args.pep_config,
+                    subcommand_args.output_dir,
+                    subcommand_args.sample_pipeline_interfaces,
+                    subcommand_args.project_pipeline_interfaces,
+                    subcommand_args.force_yes,
+                )
             )
-        )
+        else:
+            return int(looper_config_tutorial())
 
     if subcommand_name == "init_piface":
         sys.exit(int(not init_generic_pipeline()))
diff --git a/looper/utils.py b/looper/utils.py
index 9c66d33d..4cd651d1 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -21,6 +21,7 @@
 from .const import *
 from .command_models.commands import SUPPORTED_COMMANDS
 from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
+from rich.console import Console
 
 _LOGGER = getLogger(__name__)
 
@@ -535,6 +536,100 @@ def initiate_looper_config(
     print(f"Initialized looper config file: {looper_config_path}")
     return True
 
+def looper_config_tutorial():
+    # Prompt a user through configuring a .looper.yaml file for a new project.
+    # To be used in as an option for `looper init`.
+
+    console = Console()
+    console.clear()
+    console.rule(f"\n[dark_goldenrod]Looper initialization[/dark_goldenrod]")
+
+    looper_cfg_path = ".looper.yaml"  # not changeable
+
+    if os.path.exists(looper_cfg_path):
+        console.print(f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]")
+        raise SystemExit
+
+    cfg = {}
+
+    console.print("This utility will walk you through creating a [yellow].looper.yaml[/yellow] file.")
+    console.print("See [yellow]`looper init --help`[/yellow] for details.")
+    console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
+    console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
+
+    console.input("> ... ")
+
+    DEFAULTS = {  # What you get if you just press enter
+        "pep_config": "databio/example",
+        "output_dir": "results",
+        "piface_path": "pipeline_interface.yaml",
+        "project_name": os.path.basename(os.getcwd()),
+    }
+
+    creating = True
+
+    while creating:
+        cfg["project_name"] = (
+                console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >") or DEFAULTS["project_name"]
+        )
+
+        cfg["pep_config"] = (
+                console.input(f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >")
+                or DEFAULTS["pep_config"]
+        )
+
+        if not os.path.exists(cfg["pep_config"]):
+            console.print(f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'")
+
+        cfg["output_dir"] = (
+                console.input(f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >")
+                or DEFAULTS["output_dir"]
+        )
+
+        # TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline
+        # but this is not the only way you could configure things.
+
+        piface_path = (
+                console.input("Path to sample pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >")
+                or DEFAULTS["piface_path"]
+        )
+        console.print("\n")
+
+        console.print(            f"""\
+    [yellow]pep_config:[/yellow] {cfg['pep_config']}
+    [yellow]output_dir:[/yellow] {cfg['output_dir']}
+    [yellow]pipeline_interfaces:[/yellow]
+      - {piface_path}
+    """)
+
+        console.print("[bold]Does this look good?[/bold]  [bold green]Y[/bold green]/[red]n[/red]...")
+        selection = None
+        while selection not in ['y','Y','n','N']:
+            selection = console.input("\nSelection: ")
+        if selection in ['n', 'N']:
+            console.print("Starting over...")
+            pass
+        if selection in ['y', 'Y']:
+            creating=False
+
+    if not os.path.exists(piface_path):
+        console.print(f"[bold red]Warning:[/bold red] file does not exist at [yellow]{piface_path}[/yellow]")
+
+    console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
+    # print(f"PEP path: {cfg['pep_config']}")
+    # print(f"Pipeline interface path: {piface_path}")
+
+    with open(looper_cfg_path, "w") as fp:
+        fp.write(
+            f"""\
+    pep_config: {cfg['pep_config']}
+    output_dir: {cfg['output_dir']}
+    pipeline_interfaces:
+      - {piface_path}
+    """
+        )
+
+    return True
 
 def determine_pipeline_type(piface_path: str, looper_config_path: str):
     """
diff --git a/looper_init.py b/looper_init.py
deleted file mode 100644
index 9d7a3c5f..00000000
--- a/looper_init.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# A simple utility, to be run in the root of a project, to prompt a user through
-# configuring a .looper.yaml file for a new project. To be used as `looper init`.
-
-import os
-
-cfg = {}
-
-print("This utility will walk you through creating a .looper.yaml file.")
-print("See `looper init --help` for details.")
-print("Use `looper run` afterwards to run the pipeline.")
-print("Press ^C at any time to quit.\n")
-
-looper_cfg_path = ".looper.yaml"  # not changeable
-
-if os.path.exists(looper_cfg_path):
-    print(f"File exists at '{looper_cfg_path}'. Delete it to re-initialize.")
-    raise SystemExit
-
-DEFAULTS = {  # What you get if you just press enter
-    "pep_config": "databio/example",
-    "output_dir": "results",
-    "piface_path": "pipeline_interface.yaml",
-    "project_name": os.path.basename(os.getcwd()),
-}
-
-
-cfg["project_name"] = (
-    input(f"Project name: ({DEFAULTS['project_name']}) ") or DEFAULTS["project_name"]
-)
-
-cfg["pep_config"] = (
-    input(f"Registry path or file path to PEP: ({DEFAULTS['pep_config']}) ")
-    or DEFAULTS["pep_config"]
-)
-
-if not os.path.exists(cfg["pep_config"]):
-    print(f"Warning: PEP file does not exist at '{cfg['pep_config']}'")
-
-cfg["output_dir"] = (
-    input(f"Path to output directory: ({DEFAULTS['output_dir']}) ")
-    or DEFAULTS["output_dir"]
-)
-
-# TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline
-# but this is not the only way you could configure things.
-
-piface_path = (
-    input("Path to sample pipeline interface: (pipeline_interface.yaml) ")
-    or DEFAULTS["piface_path"]
-)
-
-if not os.path.exists(piface_path):
-    print(f"Warning: file does not exist at {piface_path}")
-
-print(f"Writing config file to {looper_cfg_path}")
-print(f"PEP path: {cfg['pep_config']}")
-print(f"Pipeline interface path: {piface_path}")
-
-
-with open(looper_cfg_path, "w") as fp:
-    fp.write(
-        f"""\
-pep_config: {cfg['pep_config']}
-output_dir: {cfg['output_dir']}
-pipeline_interfaces:
-  sample: {piface_path}
-"""
-    )
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index d120722e..948740ac 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -588,7 +588,7 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_not_in_any_files(subs_list, "testin_mem")
 
-
+@pytest.mark.skip(reason="This functionality requires input from the user. Causing pytest to error if run without -s flag")
 class TestLooperConfig:
 
     def test_init_config_file(self, prep_temp_pep):

From db29fd9a67cb11769b906af74ff1cd40488624df Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:36:09 -0400
Subject: [PATCH 09/56] lint

---
 looper/cli_pydantic.py       | 11 ++++---
 looper/utils.py              | 59 +++++++++++++++++++++++++-----------
 tests/smoketests/test_run.py |  5 ++-
 3 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index a9b762c3..7037616d 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -53,7 +53,8 @@
     init_generic_pipeline,
     read_yaml_file,
     inspect_looper_config_file,
-    is_PEP_file_type, looper_config_tutorial,
+    is_PEP_file_type,
+    looper_config_tutorial,
 )
 
 from typing import List, Tuple
@@ -127,13 +128,15 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         console = Console()
         console.clear()
         console.rule(f"\n[dark_goldenrod]Looper initialization[/dark_goldenrod]")
-        console.print("[bold]Would you like to follow a guided tutorial?[/bold]  [green]Y[/green] / [red]n[/red]...")
+        console.print(
+            "[bold]Would you like to follow a guided tutorial?[/bold]  [green]Y[/green] / [red]n[/red]..."
+        )
 
         selection = None
-        while selection not in ['y','Y','n','N']:
+        while selection not in ["y", "Y", "n", "N"]:
             selection = console.input("\nSelection: ")
 
-        if selection in ['n', 'N']:
+        if selection in ["n", "N"]:
 
             return int(
                 not initiate_looper_config(
diff --git a/looper/utils.py b/looper/utils.py
index 4cd651d1..b9804b29 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -536,6 +536,7 @@ def initiate_looper_config(
     print(f"Initialized looper config file: {looper_config_path}")
     return True
 
+
 def looper_config_tutorial():
     # Prompt a user through configuring a .looper.yaml file for a new project.
     # To be used in as an option for `looper init`.
@@ -547,12 +548,16 @@ def looper_config_tutorial():
     looper_cfg_path = ".looper.yaml"  # not changeable
 
     if os.path.exists(looper_cfg_path):
-        console.print(f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]")
+        console.print(
+            f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]"
+        )
         raise SystemExit
 
     cfg = {}
 
-    console.print("This utility will walk you through creating a [yellow].looper.yaml[/yellow] file.")
+    console.print(
+        "This utility will walk you through creating a [yellow].looper.yaml[/yellow] file."
+    )
     console.print("See [yellow]`looper init --help`[/yellow] for details.")
     console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
     console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
@@ -570,50 +575,67 @@ def looper_config_tutorial():
 
     while creating:
         cfg["project_name"] = (
-                console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >") or DEFAULTS["project_name"]
+            console.input(
+                f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >"
+            )
+            or DEFAULTS["project_name"]
         )
 
         cfg["pep_config"] = (
-                console.input(f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >")
-                or DEFAULTS["pep_config"]
+            console.input(
+                f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
+            )
+            or DEFAULTS["pep_config"]
         )
 
         if not os.path.exists(cfg["pep_config"]):
-            console.print(f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'")
+            console.print(
+                f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
+            )
 
         cfg["output_dir"] = (
-                console.input(f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >")
-                or DEFAULTS["output_dir"]
+            console.input(
+                f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
+            )
+            or DEFAULTS["output_dir"]
         )
 
         # TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline
         # but this is not the only way you could configure things.
 
         piface_path = (
-                console.input("Path to sample pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >")
-                or DEFAULTS["piface_path"]
+            console.input(
+                "Path to sample pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
+            )
+            or DEFAULTS["piface_path"]
         )
         console.print("\n")
 
-        console.print(            f"""\
+        console.print(
+            f"""\
     [yellow]pep_config:[/yellow] {cfg['pep_config']}
     [yellow]output_dir:[/yellow] {cfg['output_dir']}
     [yellow]pipeline_interfaces:[/yellow]
       - {piface_path}
-    """)
+    """
+        )
 
-        console.print("[bold]Does this look good?[/bold]  [bold green]Y[/bold green]/[red]n[/red]...")
+        console.print(
+            "[bold]Does this look good?[/bold]  [bold green]Y[/bold green]/[red]n[/red]..."
+        )
         selection = None
-        while selection not in ['y','Y','n','N']:
+        while selection not in ["y", "Y", "n", "N"]:
             selection = console.input("\nSelection: ")
-        if selection in ['n', 'N']:
+        if selection in ["n", "N"]:
             console.print("Starting over...")
             pass
-        if selection in ['y', 'Y']:
-            creating=False
+        if selection in ["y", "Y"]:
+            creating = False
 
     if not os.path.exists(piface_path):
-        console.print(f"[bold red]Warning:[/bold red] file does not exist at [yellow]{piface_path}[/yellow]")
+        console.print(
+            f"[bold red]Warning:[/bold red] file does not exist at [yellow]{piface_path}[/yellow]"
+        )
 
     console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
     # print(f"PEP path: {cfg['pep_config']}")
@@ -631,6 +653,7 @@ def looper_config_tutorial():
 
     return True
 
+
 def determine_pipeline_type(piface_path: str, looper_config_path: str):
     """
     Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 948740ac..e1da06fd 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -588,7 +588,10 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_not_in_any_files(subs_list, "testin_mem")
 
-@pytest.mark.skip(reason="This functionality requires input from the user. Causing pytest to error if run without -s flag")
+
+@pytest.mark.skip(
+    reason="This functionality requires input from the user. Causing pytest to error if run without -s flag"
+)
 class TestLooperConfig:
 
     def test_init_config_file(self, prep_temp_pep):

From acc02abd5a2a3f9618efcac35ab4c3a542883ef2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:16:45 -0400
Subject: [PATCH 10/56] add changes to init_piface, change colors for better
 contrast

---
 looper/cli_pydantic.py |  5 +--
 looper/utils.py        | 78 +++++++++++++++++++++++++++++++-----------
 2 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7037616d..cb619b37 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -127,7 +127,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
         console = Console()
         console.clear()
-        console.rule(f"\n[dark_goldenrod]Looper initialization[/dark_goldenrod]")
+        console.rule(f"\n[magenta]Looper initialization[/magenta]")
         console.print(
             "[bold]Would you like to follow a guided tutorial?[/bold]  [green]Y[/green] / [red]n[/red]..."
         )
@@ -137,7 +137,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             selection = console.input("\nSelection: ")
 
         if selection in ["n", "N"]:
-
+            console.clear()
             return int(
                 not initiate_looper_config(
                     dotfile_path(),
@@ -149,6 +149,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 )
             )
         else:
+            console.clear()
             return int(looper_config_tutorial())
 
     if subcommand_name == "init_piface":
diff --git a/looper/utils.py b/looper/utils.py
index b9804b29..67192711 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -22,6 +22,7 @@
 from .command_models.commands import SUPPORTED_COMMANDS
 from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
 from rich.console import Console
+from rich.pretty import pprint
 
 _LOGGER = getLogger(__name__)
 
@@ -407,6 +408,8 @@ def init_generic_pipeline():
     """
     Create generic pipeline interface
     """
+    console = Console()
+
     try:
         os.makedirs("pipeline")
     except FileExistsError:
@@ -418,21 +421,26 @@ def init_generic_pipeline():
     # Create Generic Pipeline Interface
     generic_pipeline_dict = {
         "pipeline_name": "default_pipeline_name",
-        "pipeline_type": "sample",
         "output_schema": "output_schema.yaml",
-        "var_templates": {"pipeline": "{looper.piface_dir}/pipeline.sh"},
-        "command_template": "{pipeline.var_templates.pipeline} {sample.file} "
-        "--output-parent {looper.sample_output_folder}",
+        "var_templates": {"pipeline": "{looper.piface_dir}/count_lines.sh"},
+        "sample_interface": {
+            "command_template": "{pipeline.var_templates.pipeline} {sample.file} "
+            "--output-parent {looper.sample_output_folder}"
+        },
     }
 
+    console.rule(f"\n[magenta]Pipeline Interface[/magenta]")
     # Write file
     if not os.path.exists(dest_file):
+        pprint(generic_pipeline_dict, expand_all=True)
         with open(dest_file, "w") as file:
             yaml.dump(generic_pipeline_dict, file)
-        print(f"Pipeline interface successfully created at: {dest_file}")
+        console.print(
+            f"Pipeline interface successfully created at: [yellow]{dest_file}[/yellow]"
+        )
     else:
-        print(
-            f"Pipeline interface file already exists `{dest_file}`. Skipping creation.."
+        console.print(
+            f"Pipeline interface file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
         )
 
     # Create Generic Output Schema
@@ -446,14 +454,22 @@ def init_generic_pipeline():
             }
         },
     }
+
+    console.rule(f"\n[magenta]Output Schema[/magenta]")
     # Write file
     if not os.path.exists(dest_file):
+        pprint(generic_output_schema_dict, expand_all=True)
         with open(dest_file, "w") as file:
             yaml.dump(generic_output_schema_dict, file)
-        print(f"Output schema successfully created at: {dest_file}")
+        console.print(
+            f"Output schema successfully created at: [yellow]{dest_file}[/yellow]"
+        )
     else:
-        print(f"Output schema file already exists `{dest_file}`. Skipping creation..")
+        console.print(
+            f"Output schema file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
+        )
 
+    console.rule(f"\n[magenta]Example Pipeline Shell Script[/magenta]")
     # Create Generic countlines.sh
     dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
     shell_code = """#!/bin/bash
@@ -462,11 +478,16 @@ def init_generic_pipeline():
 echo "Number of lines: $linecount"
     """
     if not os.path.exists(dest_file):
+        console.print(shell_code)
         with open(dest_file, "w") as file:
             file.write(shell_code)
-        print(f"count_lines.sh successfully created at: {dest_file}")
+        console.print(
+            f"count_lines.sh successfully created at: [yellow]{dest_file}[/yellow]"
+        )
     else:
-        print(f"count_lines.sh file already exists `{dest_file}`. Skipping creation..")
+        console.print(
+            f"count_lines.sh file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
+        )
 
     return True
 
@@ -501,8 +522,14 @@ def initiate_looper_config(
     :param bool force: whether the existing file should be overwritten
     :return bool: whether the file was initialized
     """
+    console = Console()
+    console.clear()
+    console.rule(f"\n[magenta]Looper initialization[/magenta]")
+
     if os.path.exists(looper_config_path) and not force:
-        print(f"Can't initialize, file exists: {looper_config_path}")
+        console.print(
+            f"[red]Can't initialize, file exists:[/red] [yellow]{looper_config_path}[/yellow]"
+        )
         return False
 
     if pep_path:
@@ -522,18 +549,29 @@ def initiate_looper_config(
     if not output_dir:
         output_dir = "."
 
+    if sample_pipeline_interfaces is None or sample_pipeline_interfaces == []:
+        sample_pipeline_interfaces = "pipeline_interface1.yaml"
+
+    if project_pipeline_interfaces is None or project_pipeline_interfaces == []:
+        project_pipeline_interfaces = "pipeline_interface2.yaml"
+
     looper_config_dict = {
         "pep_config": os.path.relpath(pep_path),
         "output_dir": output_dir,
-        "pipeline_interfaces": {
-            "sample": sample_pipeline_interfaces,
-            "project": project_pipeline_interfaces,
-        },
+        "pipeline_interfaces": [
+            sample_pipeline_interfaces,
+            project_pipeline_interfaces,
+        ],
     }
 
+    pprint(looper_config_dict, expand_all=True)
+
     with open(looper_config_path, "w") as dotfile:
         yaml.dump(looper_config_dict, dotfile)
-    print(f"Initialized looper config file: {looper_config_path}")
+    console.print(
+        f"Initialized looper config file: [yellow]{looper_config_path}[/yellow]"
+    )
+
     return True
 
 
@@ -543,7 +581,7 @@ def looper_config_tutorial():
 
     console = Console()
     console.clear()
-    console.rule(f"\n[dark_goldenrod]Looper initialization[/dark_goldenrod]")
+    console.rule(f"\n[magenta]Looper initialization[/magenta]")
 
     looper_cfg_path = ".looper.yaml"  # not changeable
 
@@ -567,7 +605,7 @@ def looper_config_tutorial():
     DEFAULTS = {  # What you get if you just press enter
         "pep_config": "databio/example",
         "output_dir": "results",
-        "piface_path": "pipeline_interface.yaml",
+        "piface_path": "pipeline/pipeline_interface.yaml",
         "project_name": os.path.basename(os.getcwd()),
     }
 
@@ -634,7 +672,7 @@ def looper_config_tutorial():
 
     if not os.path.exists(piface_path):
         console.print(
-            f"[bold red]Warning:[/bold red] file does not exist at [yellow]{piface_path}[/yellow]"
+            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]\nUse command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
         )
 
     console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")

From 0ffb6349a1f911e97e1ff44816ee839f78a34856 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 12:48:22 -0400
Subject: [PATCH 11/56] Update looper/cli_pydantic.py

Co-authored-by: Oleksandr <sasha99250@gmail.com>
---
 looper/cli_pydantic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index cb619b37..7ce1dacb 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -133,8 +133,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         )
 
         selection = None
-        while selection not in ["y", "Y", "n", "N"]:
-            selection = console.input("\nSelection: ")
+        while selection not in ["y", "n"]:
+            selection = console.input("\nSelection: ").lower().strip()
 
         if selection in ["n", "N"]:
             console.clear()

From f63597cb4e22b31bd6f68b05fcf038e91ae8d2eb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 12:48:32 -0400
Subject: [PATCH 12/56] Update looper/cli_pydantic.py

Co-authored-by: Oleksandr <sasha99250@gmail.com>
---
 looper/cli_pydantic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7ce1dacb..42078b84 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -136,7 +136,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         while selection not in ["y", "n"]:
             selection = console.input("\nSelection: ").lower().strip()
 
-        if selection in ["n", "N"]:
+        if selection == "n":
             console.clear()
             return int(
                 not initiate_looper_config(

From a24e0e8d4f1e9295ac6c97a8cfd2d8450bb0bb86 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 12:52:16 -0400
Subject: [PATCH 13/56] simplify selection options

---
 looper/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 67192711..79bbd51e 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -662,12 +662,12 @@ def looper_config_tutorial():
             "[bold]Does this look good?[/bold]  [bold green]Y[/bold green]/[red]n[/red]..."
         )
         selection = None
-        while selection not in ["y", "Y", "n", "N"]:
-            selection = console.input("\nSelection: ")
-        if selection in ["n", "N"]:
+        while selection not in ["y", "n"]:
+            selection = console.input("\nSelection: ").lower().strip()
+        if selection == "n":
             console.print("Starting over...")
             pass
-        if selection in ["y", "Y"]:
+        if selection == "y":
             creating = False
 
     if not os.path.exists(piface_path):

From 9d4551e6ba8aba88a7cd2d1cf783b62d91356bc6 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 12:54:18 -0400
Subject: [PATCH 14/56] add docstrings

---
 looper/utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 79bbd51e..2da99559 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -576,8 +576,11 @@ def initiate_looper_config(
 
 
 def looper_config_tutorial():
-    # Prompt a user through configuring a .looper.yaml file for a new project.
-    # To be used in as an option for `looper init`.
+    """
+    Prompt a user through configuring a .looper.yaml file for a new project.
+
+    :return bool: whether the file was initialized
+    """
 
     console = Console()
     console.clear()

From cc1b1ad5eb0d90f7b1056625094004a882a059a0 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 13:03:43 -0400
Subject: [PATCH 15/56] fix looper config formatting on write

---
 looper/utils.py | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 2da99559..328d3402 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -641,9 +641,6 @@ def looper_config_tutorial():
             or DEFAULTS["output_dir"]
         )
 
-        # TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline
-        # but this is not the only way you could configure things.
-
         piface_path = (
             console.input(
                 "Path to sample pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
@@ -679,18 +676,14 @@ def looper_config_tutorial():
         )
 
     console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
-    # print(f"PEP path: {cfg['pep_config']}")
-    # print(f"Pipeline interface path: {piface_path}")
+
+    looper_config_dict = {}
+    looper_config_dict["pep_config"] = cfg["pep_config"]
+    looper_config_dict["output_dir"] = cfg["output_dir"]
+    looper_config_dict["pipeline_interfaces"] = [piface_path]
 
     with open(looper_cfg_path, "w") as fp:
-        fp.write(
-            f"""\
-    pep_config: {cfg['pep_config']}
-    output_dir: {cfg['output_dir']}
-    pipeline_interfaces:
-      - {piface_path}
-    """
-        )
+        yaml.dump(looper_config_dict, fp)
 
     return True
 

From 0a90e9af577edccea9cdf71cade96718e0a85d19 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 14:24:34 -0400
Subject: [PATCH 16/56] add PipelineInterfaceConfigError

---
 looper/project.py |  1 -
 looper/utils.py   | 13 ++++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/looper/project.py b/looper/project.py
index e485bfa5..d77dc368 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -519,7 +519,6 @@ def _create_pipestat_config(self, piface, pipeline_type):
 
         # Warn user if there is a mismatch in pipeline_names from sources!!!
         if pipeline_name != output_schema_pipeline_name:
-            # TODO Raise exception here.
             _LOGGER.warning(
                 msg=f"Pipeline name mismatch detected. Pipeline interface: {pipeline_name}  Output schema: {output_schema_pipeline_name}  Defaulting to pipeline_interface value."
             )
diff --git a/looper/utils.py b/looper/utils.py
index 328d3402..f73e6fb7 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -717,8 +717,9 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
         pipeline_types.append(PipelineLevel.PROJECT.value)
 
     if pipeline_types == []:
-        # TODO WARN USER THEY MUST GIVE EITHER A SAMPLE OR PROJECT INTERFACE
-        return None, None
+        raise PipelineInterfaceConfigError(
+            f"sample_interface and/or project_interface must be defined in each pipeline interface."
+        )
 
     return pipeline_types, piface_path
 
@@ -767,6 +768,7 @@ def read_looper_config_file(looper_config_path: str) -> dict:
         dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
 
         all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY)
+
         sample_pifaces = []
         project_pifaces = []
         if isinstance(all_pipeline_interfaces, str):
@@ -775,14 +777,7 @@ def read_looper_config_file(looper_config_path: str) -> dict:
             pipeline_types, piface_path = determine_pipeline_type(
                 piface, looper_config_path
             )
-            # if pipeline_types is None:
-            #     raise PipelineInterfaceConfigError(
-            #         f"'sample_interface and/or project_interface must be defined in each pipeline interface."
-            #     )
-            # This will append the same, consolidated piface to two different lists
-            # In reality only the command templates are the differentiator
             if pipeline_types is not None:
-                # TODO should we raise an exception here? I guess you can amend samples with interfaces...
                 if PipelineLevel.SAMPLE.value in pipeline_types:
                     sample_pifaces.append(piface_path)
                 if PipelineLevel.PROJECT.value in pipeline_types:

From 09c06650026e527737c372302ee567ac2bd7e0b6 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 1 Jul 2024 14:55:46 -0400
Subject: [PATCH 17/56] update version and changelog

---
 docs/changelog.md      | 6 ++++++
 looper/_version.py     | 4 ++--
 looper/cli_pydantic.py | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 3b619423..a80b696c 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,6 +2,12 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [2.0.0] -- 2024-08-XX
+
+### Changed
+- refactored some CLI arguments  [455](https://github.com/pepkit/looper/issues/455)
+- consolidate sample and project interfaces under a single pipeline interface [493](https://github.com/pepkit/looper/issues/493)
+- added guided tutorial for initializing looper config file [466](https://github.com/pepkit/looper/issues/466)
 
 ## [1.9.0] -- 2024-06-26
 
diff --git a/looper/_version.py b/looper/_version.py
index 65d6992c..33ca05a0 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1,2 +1,2 @@
-__version__ = "1.9.0"
-# You must change the version in parser = pydantic2_argparse.ArgumentParser in cli_pydantic.py!!!
+__version__ = "2.0.0a1"
+# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 43ecfe25..cde67f98 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -346,7 +346,7 @@ def main(test_args=None) -> None:
         prog="looper",
         description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="1.9.0",
+        version="2.0.0a1",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)

From c344f92370856c8fbbc70722642b7fb709099b60 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 3 Jul 2024 07:14:39 -0400
Subject: [PATCH 18/56] add ability to init generic piface during config if it
 does not exist

---
 looper/utils.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index f73e6fb7..2bc532f8 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -672,8 +672,20 @@ def looper_config_tutorial():
 
     if not os.path.exists(piface_path):
         console.print(
-            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]\nUse command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
+            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]"
         )
+        console.print(
+            "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
+        )
+        selection = None
+        while selection not in ["y", "n"]:
+            selection = console.input("\nSelection: ").lower().strip()
+        if selection == "n":
+            console.print(
+                "Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
+            )
+        if selection == "y":
+            init_generic_pipeline()
 
     console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
 

From 487e8331917938e79d78c1cea7414e0a92fd33fe Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 3 Jul 2024 07:37:28 -0400
Subject: [PATCH 19/56] add accepting multiple pifaces

---
 looper/utils.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 2bc532f8..aaa8bc50 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -641,12 +641,22 @@ def looper_config_tutorial():
             or DEFAULTS["output_dir"]
         )
 
-        piface_path = (
-            console.input(
-                "Path to sample pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
+        add_more_pifaces = True
+        piface_paths = []
+        while add_more_pifaces:
+            piface_path = (
+                console.input(
+                    "Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
+                )
+                or None
             )
-            or DEFAULTS["piface_path"]
-        )
+            if piface_path is None:
+                if piface_paths == []:
+                    piface_paths.append(DEFAULTS["piface_path"])
+                add_more_pifaces = False
+            else:
+                piface_paths.append(piface_path)
+
         console.print("\n")
 
         console.print(
@@ -654,7 +664,7 @@ def looper_config_tutorial():
     [yellow]pep_config:[/yellow] {cfg['pep_config']}
     [yellow]output_dir:[/yellow] {cfg['output_dir']}
     [yellow]pipeline_interfaces:[/yellow]
-      - {piface_path}
+      - {piface_paths}
     """
         )
 
@@ -670,9 +680,9 @@ def looper_config_tutorial():
         if selection == "y":
             creating = False
 
-    if not os.path.exists(piface_path):
+    if not os.path.exists(piface_paths[0]):
         console.print(
-            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]"
+            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_paths[0]}[/yellow]"
         )
         console.print(
             "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
@@ -692,7 +702,7 @@ def looper_config_tutorial():
     looper_config_dict = {}
     looper_config_dict["pep_config"] = cfg["pep_config"]
     looper_config_dict["output_dir"] = cfg["output_dir"]
-    looper_config_dict["pipeline_interfaces"] = [piface_path]
+    looper_config_dict["pipeline_interfaces"] = [piface_paths]
 
     with open(looper_cfg_path, "w") as fp:
         yaml.dump(looper_config_dict, fp)

From e5a8e7f3dc133a213c6125950e664f2db979484a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 3 Jul 2024 15:24:40 -0400
Subject: [PATCH 20/56] allow generating generic pifaces for many pifaces

---
 looper/utils.py | 71 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 23 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index aaa8bc50..827d67bf 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -404,19 +404,29 @@ def _get_subcommand_args(subcommand_name, parser_args):
     return args
 
 
-def init_generic_pipeline():
+def init_generic_pipeline(pipelinepath: Optional[str] = None):
     """
     Create generic pipeline interface
     """
     console = Console()
 
-    try:
-        os.makedirs("pipeline")
-    except FileExistsError:
-        pass
-
     # Destination one level down from CWD in pipeline folder
-    dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
+    if not pipelinepath:
+        try:
+            os.makedirs("pipeline")
+        except FileExistsError:
+            pass
+
+        dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
+    else:
+        if os.path.isabs(pipelinepath):
+            dest_file = pipelinepath
+        else:
+            dest_file = os.path.join(os.getcwd(), os.path.relpath(pipelinepath))
+        try:
+            os.makedirs(os.path.dirname(dest_file))
+        except FileExistsError:
+            pass
 
     # Create Generic Pipeline Interface
     generic_pipeline_dict = {
@@ -433,18 +443,27 @@ def init_generic_pipeline():
     # Write file
     if not os.path.exists(dest_file):
         pprint(generic_pipeline_dict, expand_all=True)
+
         with open(dest_file, "w") as file:
             yaml.dump(generic_pipeline_dict, file)
+
         console.print(
             f"Pipeline interface successfully created at: [yellow]{dest_file}[/yellow]"
         )
+
     else:
         console.print(
             f"Pipeline interface file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
         )
 
     # Create Generic Output Schema
-    dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
+    if not pipelinepath:
+        dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
+    else:
+        dest_file = os.path.join(
+            os.path.dirname(dest_file), LOOPER_GENERIC_OUTPUT_SCHEMA
+        )
+
     generic_output_schema_dict = {
         "pipeline_name": "default_pipeline_name",
         "samples": {
@@ -471,7 +490,12 @@ def init_generic_pipeline():
 
     console.rule(f"\n[magenta]Example Pipeline Shell Script[/magenta]")
     # Create Generic countlines.sh
-    dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
+
+    if not pipelinepath:
+        dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
+    else:
+        dest_file = os.path.join(os.path.dirname(dest_file), LOOPER_GENERIC_COUNT_LINES)
+
     shell_code = """#!/bin/bash
 linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
 pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
@@ -680,22 +704,23 @@ def looper_config_tutorial():
         if selection == "y":
             creating = False
 
-    if not os.path.exists(piface_paths[0]):
-        console.print(
-            f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_paths[0]}[/yellow]"
-        )
-        console.print(
-            "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
-        )
-        selection = None
-        while selection not in ["y", "n"]:
-            selection = console.input("\nSelection: ").lower().strip()
-        if selection == "n":
+    for piface_path in piface_paths:
+        if not os.path.exists(piface_path):
             console.print(
-                "Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
+                f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]"
             )
-        if selection == "y":
-            init_generic_pipeline()
+            console.print(
+                "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
+            )
+            selection = None
+            while selection not in ["y", "n"]:
+                selection = console.input("\nSelection: ").lower().strip()
+            if selection == "n":
+                console.print(
+                    "Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
+                )
+            if selection == "y":
+                init_generic_pipeline(pipelinepath=piface_path)
 
     console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
 

From f64a4f077e875418b81afbf8773878a67e84bb95 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 18 Jul 2024 13:01:40 -0400
Subject: [PATCH 21/56] update req peppy>=0.40.4

---
 requirements/requirements-all.txt | 2 +-
 tests/smoketests/test_run.py      | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index dd40fd1d..77cb7728 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
 pipestat>=0.9.2
-peppy>=0.40.2
+peppy>=0.40.4
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.8.1a1
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index e1da06fd..59eede55 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -265,7 +265,7 @@ def test_looper_toggle(self, prep_temp_pep):
         project_config_path = get_project_config_path(tp)
 
         with mod_yaml_data(project_config_path) as project_config_data:
-            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][SAMPLE_TOGGLE_ATTR] = 0
+            project_config_data[SAMPLE_MODS_KEY][APPEND_KEY][SAMPLE_TOGGLE_ATTR] = 0
 
         x = test_args_expansion(tp, "run")
         x.pop(-1)  # remove dry run for this test
@@ -286,7 +286,7 @@ def test_cmd_extra_sample(self, prep_temp_pep, arg):
         project_config_path = get_project_config_path(tp)
 
         with mod_yaml_data(project_config_path) as project_config_data:
-            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+            project_config_data[SAMPLE_MODS_KEY][APPEND_KEY]["command_extra"] = arg
         x = test_args_expansion(tp, "run")
         try:
             main(test_args=x)
@@ -307,7 +307,7 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         project_config_path = get_project_config_path(tp)
 
         with mod_yaml_data(project_config_path) as project_config_data:
-            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+            project_config_data[SAMPLE_MODS_KEY][APPEND_KEY]["command_extra"] = arg
 
         x = test_args_expansion(tp, "run", ["--command-extra-override='different'"])
         try:

From a264e804fa171a7c165bca5bb240e6356e33557f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Aug 2024 10:33:27 -0400
Subject: [PATCH 22/56] fix #512

---
 looper/cli_divvy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/cli_divvy.py b/looper/cli_divvy.py
index 0c152e25..10c03436 100644
--- a/looper/cli_divvy.py
+++ b/looper/cli_divvy.py
@@ -126,7 +126,7 @@ def main():
     _LOGGER.debug("Divvy config: {}".format(args.config))
     divcfg = select_divvy_config(args.config)
     _LOGGER.info("Using divvy config: {}".format(divcfg))
-    dcc = ComputingConfiguration(filepath=divcfg)
+    dcc = ComputingConfiguration.from_yaml_file(filepath=divcfg)
 
     if args.command == "list":
         # Output header via logger and content via print so the user can

From 4c9549bb1fa62d23e50107ebf12b6f946b21565c Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Aug 2024 10:40:58 -0400
Subject: [PATCH 23/56] fix #511

---
 looper/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/__main__.py b/looper/__main__.py
index 3e981655..4abdf9b3 100644
--- a/looper/__main__.py
+++ b/looper/__main__.py
@@ -5,7 +5,7 @@
 
 if __name__ == "__main__":
     try:
-        sys.exit(main())
+        main()
     except KeyboardInterrupt:
         print("Program canceled by user!")
         sys.exit(1)

From 40c663412c788992d6a1dd9e44fbb13c168d0840 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:11:57 -0400
Subject: [PATCH 24/56] add flag to looper init #514

---
 looper/cli_pydantic.py             | 11 ++---------
 looper/command_models/arguments.py |  8 ++++++++
 looper/command_models/commands.py  |  1 +
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index cde67f98..c46e9a70 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -128,15 +128,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         console = Console()
         console.clear()
         console.rule(f"\n[magenta]Looper initialization[/magenta]")
-        console.print(
-            "[bold]Would you like to follow a guided tutorial?[/bold]  [green]Y[/green] / [red]n[/red]..."
-        )
-
-        selection = None
-        while selection not in ["y", "n"]:
-            selection = console.input("\nSelection: ").lower().strip()
-
-        if selection == "n":
+        selection = subcommand_args.generic
+        if selection is True:
             console.clear()
             return int(
                 not initiate_looper_config(
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 9d495393..210d6809 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -184,6 +184,14 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Output directory",
     )
+
+    GENERIC = Argument(
+        name="generic",
+        alias="-g",
+        default=(bool, False),
+        description="Use generic looper config?",
+    )
+
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
         alias="-S",
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 8ae037e0..1df4662c 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -187,6 +187,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PEP_CONFIG.value,
         ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
         ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.GENERIC.value,
     ],
 )
 

From f3aa7bf4de520514849d930d3efdc7e77e2e932b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:21:05 -0400
Subject: [PATCH 25/56] remove pause, fix registry path warning #514

---
 looper/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 827d67bf..6fd32b91 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -627,8 +627,6 @@ def looper_config_tutorial():
     console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
     console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
 
-    console.input("> ... ")
-
     DEFAULTS = {  # What you get if you just press enter
         "pep_config": "databio/example",
         "output_dir": "results",
@@ -653,7 +651,9 @@ def looper_config_tutorial():
             or DEFAULTS["pep_config"]
         )
 
-        if not os.path.exists(cfg["pep_config"]):
+        if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path(
+            cfg["pep_config"]
+        ):
             console.print(
                 f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
             )

From f5cd9067f1e644f7aa2dfe70cd1bab37bb6a5ab4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:28:02 -0400
Subject: [PATCH 26/56] remove restarting looper_init, simply show output #514

---
 looper/utils.py | 99 ++++++++++++++++++++-----------------------------
 1 file changed, 41 insertions(+), 58 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 6fd32b91..451e080f 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -634,75 +634,58 @@ def looper_config_tutorial():
         "project_name": os.path.basename(os.getcwd()),
     }
 
-    creating = True
+    cfg["project_name"] = (
+        console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >")
+        or DEFAULTS["project_name"]
+    )
 
-    while creating:
-        cfg["project_name"] = (
-            console.input(
-                f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >"
-            )
-            or DEFAULTS["project_name"]
+    cfg["pep_config"] = (
+        console.input(
+            f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
         )
+        or DEFAULTS["pep_config"]
+    )
 
-        cfg["pep_config"] = (
-            console.input(
-                f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
-            )
-            or DEFAULTS["pep_config"]
+    if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path(
+        cfg["pep_config"]
+    ):
+        console.print(
+            f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
         )
 
-        if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path(
-            cfg["pep_config"]
-        ):
-            console.print(
-                f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
-            )
+    cfg["output_dir"] = (
+        console.input(
+            f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
+        )
+        or DEFAULTS["output_dir"]
+    )
 
-        cfg["output_dir"] = (
+    add_more_pifaces = True
+    piface_paths = []
+    while add_more_pifaces:
+        piface_path = (
             console.input(
-                f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
+                "Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
             )
-            or DEFAULTS["output_dir"]
+            or None
         )
+        if piface_path is None:
+            if piface_paths == []:
+                piface_paths.append(DEFAULTS["piface_path"])
+            add_more_pifaces = False
+        else:
+            piface_paths.append(piface_path)
 
-        add_more_pifaces = True
-        piface_paths = []
-        while add_more_pifaces:
-            piface_path = (
-                console.input(
-                    "Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
-                )
-                or None
-            )
-            if piface_path is None:
-                if piface_paths == []:
-                    piface_paths.append(DEFAULTS["piface_path"])
-                add_more_pifaces = False
-            else:
-                piface_paths.append(piface_path)
-
-        console.print("\n")
+    console.print("\n")
 
-        console.print(
-            f"""\
-    [yellow]pep_config:[/yellow] {cfg['pep_config']}
-    [yellow]output_dir:[/yellow] {cfg['output_dir']}
-    [yellow]pipeline_interfaces:[/yellow]
-      - {piface_paths}
-    """
-        )
-
-        console.print(
-            "[bold]Does this look good?[/bold]  [bold green]Y[/bold green]/[red]n[/red]..."
-        )
-        selection = None
-        while selection not in ["y", "n"]:
-            selection = console.input("\nSelection: ").lower().strip()
-        if selection == "n":
-            console.print("Starting over...")
-            pass
-        if selection == "y":
-            creating = False
+    console.print(
+        f"""\
+[yellow]pep_config:[/yellow] {cfg['pep_config']}
+[yellow]output_dir:[/yellow] {cfg['output_dir']}
+[yellow]pipeline_interfaces:[/yellow]
+  - {piface_paths}
+"""
+    )
 
     for piface_path in piface_paths:
         if not os.path.exists(piface_path):

From 70dfb23b85729a0577fcfededa8aa96321a017cf Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 08:59:39 -0400
Subject: [PATCH 27/56] fix extra dash when writing .looper.yaml #514

---
 looper/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index 451e080f..b183512c 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -710,7 +710,7 @@ def looper_config_tutorial():
     looper_config_dict = {}
     looper_config_dict["pep_config"] = cfg["pep_config"]
     looper_config_dict["output_dir"] = cfg["output_dir"]
-    looper_config_dict["pipeline_interfaces"] = [piface_paths]
+    looper_config_dict["pipeline_interfaces"] = piface_paths
 
     with open(looper_cfg_path, "w") as fp:
         yaml.dump(looper_config_dict, fp)

From 54682ae6749e1a5d2d041b62b4cf3731788fdf73 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 11:52:10 -0400
Subject: [PATCH 28/56] add shortform argument for --package

---
 looper/command_models/arguments.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 210d6809..2d6b2fbc 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -236,6 +236,7 @@ class ArgumentEnum(enum.Enum):
     )
     PACKAGE = Argument(
         name="package",
+        alias="-p",
         default=(str, None),
         description="Name of computing resource package to use",
     )

From fdac0a8f5922140294c6c3c932b4b61cc42fe332 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 12:03:20 -0400
Subject: [PATCH 29/56] make compute_packages a property and fix divvy inspect

---
 looper/cli_divvy.py | 8 +++++---
 looper/divvy.py     | 5 +++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/looper/cli_divvy.py b/looper/cli_divvy.py
index 10c03436..2677b8b1 100644
--- a/looper/cli_divvy.py
+++ b/looper/cli_divvy.py
@@ -142,11 +142,13 @@ def main():
         for pkg_name, pkg in dcc.compute_packages.items():
             if pkg_name == args.package:
                 found = True
-                with open(pkg.submission_template, "r") as f:
+                with open(pkg["submission_template"], "r") as f:
                     print(f.read())
-                _LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
+                _LOGGER.info(
+                    "Submission command is: " + pkg["submission_command"] + "\n"
+                )
                 if pkg_name == "docker":
-                    print("Docker args are: " + pkg.docker_args)
+                    print("Docker args are: " + pkg["docker_args"])
 
         if not found:
             _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
diff --git a/looper/divvy.py b/looper/divvy.py
index 38dd8fe2..49c1fd23 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -200,6 +200,11 @@ def get_active_package(self) -> YAMLConfigManager:
         """
         return self.compute
 
+    @property
+    def compute_packages(self):
+
+        return self["compute_packages"]
+
     def list_compute_packages(self):
         """
         Returns a list of available compute packages.

From 7d457a1a8018750c32fe7529084914f4bae92fc0 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 12:28:41 -0400
Subject: [PATCH 30/56] remove position based argument for divvy config, must
 use --config or run as default config

---
 looper/cli_divvy.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/looper/cli_divvy.py b/looper/cli_divvy.py
index 2677b8b1..1fa98b69 100644
--- a/looper/cli_divvy.py
+++ b/looper/cli_divvy.py
@@ -53,10 +53,10 @@ def add_subparser(cmd, description):
 
     for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
         sp.add_argument(
-            "config", nargs="?", default=None, help="Divvy configuration file."
+            "--config", nargs="?", default=None, help="Divvy configuration file."
         )
 
-    sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
+    sps["init"].add_argument("--config", default=None, help="Divvy configuration file.")
 
     for sp in [sps["inspect"]]:
         sp.add_argument(
@@ -124,7 +124,9 @@ def main():
         sys.exit(0)
 
     _LOGGER.debug("Divvy config: {}".format(args.config))
+
     divcfg = select_divvy_config(args.config)
+
     _LOGGER.info("Using divvy config: {}".format(divcfg))
     dcc = ComputingConfiguration.from_yaml_file(filepath=divcfg)
 

From 7966ec6061862d6346e87d1f105ae1469520d444 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 16:04:53 -0400
Subject: [PATCH 31/56] fix compute key value parsing under cli key in looper
 config

---
 looper/cli_pydantic.py | 33 +++++++++++++++++++--------------
 looper/utils.py        |  2 ++
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index c46e9a70..80c27f8e 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -378,20 +378,25 @@ def _proc_resources_spec(args):
         settings_data = {}
     if not spec:
         return settings_data
-    pairs = [(kv, kv.split("=")) for kv in spec]
-    bads = []
-    for orig, pair in pairs:
-        try:
-            k, v = pair
-        except ValueError:
-            bads.append(orig)
-        else:
-            settings_data[k] = v
-    if bads:
-        raise ValueError(
-            "Could not correctly parse itemized compute specification. "
-            "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
-        )
+    if isinstance(spec, list):
+        pairs = [(kv, kv.split("=")) for kv in spec]
+        bads = []
+        for orig, pair in pairs:
+            try:
+                k, v = pair
+            except ValueError:
+                bads.append(orig)
+            else:
+                settings_data[k] = v
+        if bads:
+            raise ValueError(
+                "Could not correctly parse itemized compute specification. "
+                "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
+            )
+    elif isinstance(spec, dict):
+        for key, value in spec.items():
+            settings_data[key] = value
+
     return settings_data
 
 
diff --git a/looper/utils.py b/looper/utils.py
index b183512c..ccde670b 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -324,6 +324,8 @@ def set_single_arg(argname, default_source_namespace, result_namespace):
             elif cfg_args_all is not None and argname in cfg_args_all:
                 if isinstance(cfg_args_all[argname], list):
                     r = [convert_value(i) for i in cfg_args_all[argname]]
+                elif isinstance(cfg_args_all[argname], dict):
+                    r = cfg_args_all[argname]
                 else:
                     r = convert_value(cfg_args_all[argname])
             else:

From 907037ee7beb534de85d6ef151c0919669164c3b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Aug 2024 16:15:30 -0400
Subject: [PATCH 32/56] allow compute section under cli key to be a single
 string for convenience

---
 looper/cli_pydantic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 80c27f8e..49baf7c3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -378,6 +378,10 @@ def _proc_resources_spec(args):
         settings_data = {}
     if not spec:
         return settings_data
+    if isinstance(
+        spec, str
+    ):  # compute: "partition=standard time='01-00:00:00' cores='32' mem='32000'"
+        spec = spec.split(sep=" ")
     if isinstance(spec, list):
         pairs = [(kv, kv.split("=")) for kv in spec]
         bads = []

From 08af6cc59a9e39ad24e4dca368832c53cde18e77 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 23 Aug 2024 09:37:42 -0400
Subject: [PATCH 33/56] add exception messaging for phc login #516

---
 looper/cli_pydantic.py | 30 +++++++++++++++++-------------
 looper/exceptions.py   |  9 +++++++++
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 49baf7c3..30340690 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -20,6 +20,7 @@
 import sys
 
 import logmuse
+import pephubclient.exceptions
 import pydantic_argparse
 import yaml
 from eido import inspect_project
@@ -214,19 +215,22 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             sys.exit(1)
     elif is_pephub_registry_path(subcommand_args.pep_config):
         if vars(subcommand_args)[SAMPLE_PL_ARG]:
-            p = Project(
-                amendments=subcommand_args.amend,
-                divcfg_path=divcfg,
-                runp=subcommand_name == "runp",
-                project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=subcommand_args.pep_config
-                ),
-                **{
-                    attr: getattr(subcommand_args, attr)
-                    for attr in CLI_PROJ_ATTRS
-                    if attr in subcommand_args
-                },
-            )
+            try:
+                p = Project(
+                    amendments=subcommand_args.amend,
+                    divcfg_path=divcfg,
+                    runp=subcommand_name == "runp",
+                    project_dict=PEPHubClient()._load_raw_pep(
+                        registry_path=subcommand_args.pep_config
+                    ),
+                    **{
+                        attr: getattr(subcommand_args, attr)
+                        for attr in CLI_PROJ_ATTRS
+                        if attr in subcommand_args
+                    },
+                )
+            except pephubclient.exceptions.ResponseError as e:
+                raise PEPhubException(msg=e.message)
         else:
             raise MisconfigurationException(
                 f"`sample_pipeline_interface` is missing. Provide it in the parameters."
diff --git a/looper/exceptions.py b/looper/exceptions.py
index f9cb9e0c..accab212 100644
--- a/looper/exceptions.py
+++ b/looper/exceptions.py
@@ -44,6 +44,15 @@ def __init__(self, msg):
         super(RegistryPathException, self).__init__(msg)
 
 
+class PEPhubException(LooperError):
+    """For pephub client connections"""
+
+    def __init__(self, msg):
+        super(PEPhubException, self).__init__(
+            msg + "\n You may need to login by running `phc login`"
+        )
+
+
 class DuplicatePipelineKeyException(LooperError):
     """Duplication of pipeline identifier precludes unique pipeline ref."""
 

From a01722f16a6aea48a75a3f46b2869b42cc1c7b06 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 23 Aug 2024 11:34:46 -0400
Subject: [PATCH 34/56] Fix #518

---
 looper/utils.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index ccde670b..3ec36312 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -263,31 +263,29 @@ def enrich_args_via_cfg(
     cli_modifiers=None,
 ):
     """
-    Read in a looper dotfile and set arguments.
+    Read in a looper dotfile, pep config and set arguments.
 
-    Priority order: CLI > dotfile/config > parser default
+    Priority order: CLI > dotfile/config > pep_config > parser default
 
     :param subcommand name: the name of the command used
     :param argparse.Namespace parser_args: parsed args by the original parser
-    :param argparse.Namespace aux_parser: parsed args by the a parser
+    :param argparse.Namespace aux_parser: parsed args by the argument parser
         with defaults suppressed
+    :param dict test_args: dict of args used for pytesting
+    :param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file
     :return argparse.Namespace: selected argument values
     """
+
+    # Did the user provide arguments in the PEP config?
     cfg_args_all = (
         _get_subcommand_args(subcommand_name, parser_args)
         if os.path.exists(parser_args.pep_config)
         else dict()
     )
-
-    # If user provided project-level modifiers in the looper config, they are prioritized
-    if cfg_args_all:
-        for key, value in cfg_args_all.items():
-            if getattr(parser_args, key, None):
-                new_value = getattr(parser_args, key)
-                cfg_args_all[key] = new_value
-    else:
+    if not cfg_args_all:
         cfg_args_all = {}
 
+    # Did the user provide arguments/modifiers in the looper config file?
     looper_config_cli_modifiers = None
     if cli_modifiers:
         if str(subcommand_name) in cli_modifiers:
@@ -312,6 +310,13 @@ def enrich_args_via_cfg(
     else:
         cli_args, _ = aux_parser.parse_known_args()
 
+    # If any CLI args were provided, make sure they take priority
+    if cli_args:
+        r = getattr(cli_args, subcommand_name)
+        for k, v in cfg_args_all.items():
+            if k in r:
+                cfg_args_all[k] = getattr(r, k)
+
     def set_single_arg(argname, default_source_namespace, result_namespace):
         if argname not in POSITIONAL or not hasattr(result, argname):
             if argname in cli_args:

From d9ea2adaef468dafd9fe694bde4df3a218d99a39 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 23 Aug 2024 12:24:21 -0400
Subject: [PATCH 35/56] Add better warnings for missing looper config files
 #515

---
 looper/cli_pydantic.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 30340690..00977990 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -166,11 +166,14 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             else:
                 setattr(subcommand_args, looper_config_key, looper_config_item)
 
-    except OSError:
-        parser.print_help(sys.stderr)
-        _LOGGER.warning(
-            f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
-        )
+    except OSError as e:
+        if subcommand_args.config:
+            _LOGGER.warning(
+                f"\nLooper config file does not exist at given path {subcommand_args.config}. Use looper init to create one at {looper_cfg_path}."
+            )
+        else:
+            _LOGGER.warning(e)
+
         sys.exit(1)
 
     subcommand_args = enrich_args_via_cfg(

From b29494c7396d625cbf6ec68f903bd7cc3e2e7314 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 23 Aug 2024 12:55:30 -0400
Subject: [PATCH 36/56] Add missing and malformed piface warning #515

---
 looper/utils.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index 3ec36312..bbd3219b 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -737,7 +737,15 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
 
     if piface_path is None:
         return None, None
-    piface_path = expandpath(piface_path)
+    try:
+        piface_path = expandpath(piface_path)
+    except TypeError as e:
+        _LOGGER.warning(
+            f"Pipeline interface not found at given path: {piface_path}. Type Error: "
+            + str(e)
+        )
+        return None, None
+
     if not os.path.isabs(piface_path):
         piface_path = os.path.realpath(
             os.path.join(os.path.dirname(looper_config_path), piface_path)
@@ -745,6 +753,7 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
     try:
         piface_dict = load_yaml(piface_path)
     except FileNotFoundError:
+        _LOGGER.warning(f"Pipeline interface not found at given path: {piface_path}")
         return None, None
 
     pipeline_types = []
@@ -836,6 +845,7 @@ def read_looper_config_file(looper_config_path: str) -> dict:
     for k, v in return_dict.items():
         if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG:
             # Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474
+
             pass
         if isinstance(v, str):
             v = expandpath(v)

From db8c228423119607e2b8892eb1fec4db03eeec64 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 23 Aug 2024 13:21:52 -0400
Subject: [PATCH 37/56] add clarity if config file cannot be parsed #515

---
 looper/exceptions.py |  2 +-
 looper/utils.py      | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/looper/exceptions.py b/looper/exceptions.py
index accab212..285a43ae 100644
--- a/looper/exceptions.py
+++ b/looper/exceptions.py
@@ -31,7 +31,7 @@ class SampleFailedException(LooperError):
 
 
 class MisconfigurationException(LooperError):
-    """Duplication of pipeline identifier precludes unique pipeline ref."""
+    """Looper not properly configured"""
 
     def __init__(self, key):
         super(MisconfigurationException, self).__init__(key)
diff --git a/looper/utils.py b/looper/utils.py
index bbd3219b..aa2f6300 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -17,6 +17,7 @@
 from pephubclient.constants import RegistryPath
 from pydantic import ValidationError
 from yacman import load_yaml
+from yaml.parser import ParserError
 
 from .const import *
 from .command_models.commands import SUPPORTED_COMMANDS
@@ -782,12 +783,18 @@ def read_looper_config_file(looper_config_path: str) -> dict:
     :raise MisconfigurationException: incorrect configuration.
     """
     return_dict = {}
-    with open(looper_config_path, "r") as dotfile:
-        dp_data = yaml.safe_load(dotfile)
+
+    try:
+        with open(looper_config_path, "r") as dotfile:
+            dp_data = yaml.safe_load(dotfile)
+    except ParserError as e:
+        _LOGGER.warning(
+            "Could not load looper config file due to the following exception"
+        )
+        raise ParserError(context=str(e))
 
     if PEP_CONFIG_KEY in dp_data:
         return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
-
     else:
         raise MisconfigurationException(
             f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key"

From 3a9874ef53c2a02521fd8e561aa8485bdf44ef86 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 26 Aug 2024 12:12:35 -0400
Subject: [PATCH 38/56] allow calling pephub_path directly from pipestat
 namespace, version bump pipestat to 0.10.2, Fixes #519

---
 looper/conductor.py               | 1 +
 requirements/requirements-all.txt | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 70960d1f..231a2366 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -661,6 +661,7 @@ def _set_pipestat_namespace(
                 "record_identifier": psm.record_identifier,
                 "config_file": psm.config_path,
                 "output_schema": psm.cfg["_schema_path"],
+                "pephub_path": psm.cfg["pephub_path"],
             }
             filtered_namespace = {k: v for k, v in full_namespace.items() if v}
             return YAMLConfigManager(filtered_namespace)
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 77cb7728..3db01243 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -5,7 +5,7 @@ jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
-pipestat>=0.9.2
+pipestat>=0.10.2
 peppy>=0.40.4
 pyyaml>=3.12
 rich>=9.10.0

From ffa812b1eff34ec886458d8847fed0f22b384654 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Aug 2024 12:32:32 -0400
Subject: [PATCH 39/56] fix for #511

---
 looper/cli_pydantic.py | 5 ++++-
 setup.py               | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 00977990..1a886f4b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -340,7 +340,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 _LOGGER.warning("No looper configuration was supplied.")
 
 
-def main(test_args=None) -> None:
+def main(test_args=None) -> dict:
     parser = pydantic_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
@@ -358,6 +358,9 @@ def main(test_args=None) -> None:
 
     return run_looper(args, parser, test_args=test_args)
 
+def main_cli() -> None:
+    main()
+
 
 def _proc_resources_spec(args):
     """
diff --git a/setup.py b/setup.py
index db8d9459..08c455ba 100644
--- a/setup.py
+++ b/setup.py
@@ -79,7 +79,7 @@ def get_static(name, condition=None):
     license="BSD2",
     entry_points={
         "console_scripts": [
-            "looper = looper.cli_pydantic:main",
+            "looper = looper.cli_pydantic:main_cli",
             "divvy = looper.__main__:divvy_main",
         ],
     },

From 68a1729a4d3f5b4fc3dd72294d0f4eb48c104751 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Aug 2024 12:57:42 -0400
Subject: [PATCH 40/56] Revert "fix #511"

This reverts commit 4c9549bb1fa62d23e50107ebf12b6f946b21565c.
---
 looper/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/__main__.py b/looper/__main__.py
index 4abdf9b3..3e981655 100644
--- a/looper/__main__.py
+++ b/looper/__main__.py
@@ -5,7 +5,7 @@
 
 if __name__ == "__main__":
     try:
-        main()
+        sys.exit(main())
     except KeyboardInterrupt:
         print("Program canceled by user!")
         sys.exit(1)

From d5fd36a45799d980b0ffef587f53ed8f52c8b647 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Aug 2024 13:52:32 -0400
Subject: [PATCH 41/56] Revert "add exception messaging for phc login #516"

This reverts commit 08af6cc59a9e39ad24e4dca368832c53cde18e77.
---
 looper/cli_pydantic.py | 30 +++++++++++++-----------------
 looper/exceptions.py   |  9 ---------
 2 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 1a886f4b..c57ae3ce 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -20,7 +20,6 @@
 import sys
 
 import logmuse
-import pephubclient.exceptions
 import pydantic_argparse
 import yaml
 from eido import inspect_project
@@ -218,22 +217,19 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             sys.exit(1)
     elif is_pephub_registry_path(subcommand_args.pep_config):
         if vars(subcommand_args)[SAMPLE_PL_ARG]:
-            try:
-                p = Project(
-                    amendments=subcommand_args.amend,
-                    divcfg_path=divcfg,
-                    runp=subcommand_name == "runp",
-                    project_dict=PEPHubClient()._load_raw_pep(
-                        registry_path=subcommand_args.pep_config
-                    ),
-                    **{
-                        attr: getattr(subcommand_args, attr)
-                        for attr in CLI_PROJ_ATTRS
-                        if attr in subcommand_args
-                    },
-                )
-            except pephubclient.exceptions.ResponseError as e:
-                raise PEPhubException(msg=e.message)
+            p = Project(
+                amendments=subcommand_args.amend,
+                divcfg_path=divcfg,
+                runp=subcommand_name == "runp",
+                project_dict=PEPHubClient()._load_raw_pep(
+                    registry_path=subcommand_args.pep_config
+                ),
+                **{
+                    attr: getattr(subcommand_args, attr)
+                    for attr in CLI_PROJ_ATTRS
+                    if attr in subcommand_args
+                },
+            )
         else:
             raise MisconfigurationException(
                 f"`sample_pipeline_interface` is missing. Provide it in the parameters."
diff --git a/looper/exceptions.py b/looper/exceptions.py
index 285a43ae..469f68af 100644
--- a/looper/exceptions.py
+++ b/looper/exceptions.py
@@ -44,15 +44,6 @@ def __init__(self, msg):
         super(RegistryPathException, self).__init__(msg)
 
 
-class PEPhubException(LooperError):
-    """For pephub client connections"""
-
-    def __init__(self, msg):
-        super(PEPhubException, self).__init__(
-            msg + "\n You may need to login by running `phc login`"
-        )
-
-
 class DuplicatePipelineKeyException(LooperError):
     """Duplication of pipeline identifier precludes unique pipeline ref."""
 

From d0cffcba255485e13214e631fa6e05b5817bc422 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Aug 2024 14:37:12 -0400
Subject: [PATCH 42/56] replace deprecated func

---
 looper/cli_pydantic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index c57ae3ce..4d815d91 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -221,7 +221,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
-                project_dict=PEPHubClient()._load_raw_pep(
+                project_dict=PEPHubClient().load_raw_pep(
                     registry_path=subcommand_args.pep_config
                 ),
                 **{

From 165b18363ca5bf7bac2de7bb6050245e1f8fd46d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Aug 2024 10:12:55 -0400
Subject: [PATCH 43/56] Fix divvy, remove requirement for old package, ensure
 looper is using internal module #520

---
 looper/cli_pydantic.py                | 3 +--
 looper/divvy.py                       | 6 ++++--
 requirements/requirements-all.txt     | 1 -
 tests/divvytests/test_divvy_simple.py | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 4d815d91..df888fb4 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -26,8 +26,6 @@
 from pephubclient import PEPHubClient
 from pydantic_argparse.argparse.parser import ArgumentParser
 
-from divvy import select_divvy_config
-
 from . import __version__
 
 from .command_models.arguments import ArgumentEnum
@@ -354,6 +352,7 @@ def main(test_args=None) -> dict:
 
     return run_looper(args, parser, test_args=test_args)
 
+
 def main_cli() -> None:
     main()
 
diff --git a/looper/divvy.py b/looper/divvy.py
index 49c1fd23..856219cc 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -401,11 +401,13 @@ def divvy_init(config_path, template_config_path):
         _LOGGER.error("You must specify a template config file path.")
         return
 
+    if not os.path.isabs(config_path):
+        config_path = os.path.abspath(config_path)
+
     if config_path and not os.path.exists(config_path):
-        # dcc.write(config_path)
         # Init should *also* write the templates.
         dest_folder = os.path.dirname(config_path)
-        copytree(os.path.dirname(template_config_path), dest_folder)
+        copytree(os.path.dirname(template_config_path), dest_folder, dirs_exist_ok=True)
         template_subfolder = os.path.join(dest_folder, "divvy_templates")
         _LOGGER.info("Wrote divvy templates to folder: {}".format(template_subfolder))
         new_template = os.path.join(
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 3db01243..678b318a 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -1,5 +1,4 @@
 colorama>=0.3.9
-divvy>=0.5.0
 eido>=0.2.1
 jinja2
 logmuse>=0.2.0
diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py
index f7795696..5770661f 100644
--- a/tests/divvytests/test_divvy_simple.py
+++ b/tests/divvytests/test_divvy_simple.py
@@ -4,7 +4,7 @@
 from collections import OrderedDict
 
 from yacman import YacAttMap
-from divvy import select_divvy_config
+from looper.divvy import select_divvy_config
 
 # For interactive debugging:
 # import logmuse

From f23f5aa0738bc177f823c44aec95834ac950668b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 10 Sep 2024 15:51:47 -0400
Subject: [PATCH 44/56] update peppy and eido requirements

---
 requirements/requirements-all.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 678b318a..5005921c 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -1,11 +1,11 @@
 colorama>=0.3.9
-eido>=0.2.1
+eido>=0.2.3
 jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
 pipestat>=0.10.2
-peppy>=0.40.4
+peppy>=0.40.6
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.8.1a1

From a7246deabd0e3e55899bd54fdec29623ac0dcf5e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 19 Sep 2024 11:19:55 -0400
Subject: [PATCH 45/56] update the looper init_piface output to no longer use
 var_templates

---
 looper/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index aa2f6300..5a8279bd 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -440,9 +440,8 @@ def init_generic_pipeline(pipelinepath: Optional[str] = None):
     generic_pipeline_dict = {
         "pipeline_name": "default_pipeline_name",
         "output_schema": "output_schema.yaml",
-        "var_templates": {"pipeline": "{looper.piface_dir}/count_lines.sh"},
         "sample_interface": {
-            "command_template": "{pipeline.var_templates.pipeline} {sample.file} "
+            "command_template": "{looper.piface_dir}/count_lines.sh {sample.file} "
             "--output-parent {looper.sample_output_folder}"
         },
     }

From c32c9249ffaf44958a829248cb51cb0179d03c7a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 30 Sep 2024 15:10:08 -0400
Subject: [PATCH 46/56] update changelog and version for 2.0.0a2 prerelease

---
 docs/changelog.md      | 30 ++++++++++++++++++++++++++----
 looper/_version.py     |  2 +-
 looper/cli_pydantic.py |  2 +-
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index a80b696c..662481f5 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,12 +2,34 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [2.0.0] -- 2024-08-XX
+## [2.0.0a2] -- 2024-09-30
+
+This release breaks backwards compatibility for Looper versions < 2.0.0
+
+### Fixed
+- divvy init [#520](https://github.com/pepkit/looper/issues/520)
+- replaced deprecated PEPHubClient function, `_load_raw_pep` with `.load_raw_pep`
+- looper cli parameters now take priority as originally intended [#518](https://github.com/pepkit/looper/issues/518)
+- fix divvy inspect
+- remove printed dictionary at looper finish [#511](https://github.com/pepkit/looper/issues/511)
+
+### Changed
+- `--looper-config` is now `--config`, `-c`. [#455](https://github.com/pepkit/looper/issues/455)
+- A pipeline interface now consolidates a `sample_interface` and a `project_interface` [#493](https://github.com/pepkit/looper/issues/493)
+- Updated documentation for Looper 2.0.0, removing previous versions [pepspec PR #34](https://github.com/pepkit/pepspec/pull/34)
+- remove position based argument for divvy config, must use --config or run as default config
+
+
+### Added
+- `looper init` tutorial [#466](https://github.com/pepkit/looper/issues/466)
+- looper config allows for `pephub_path` in pipestat config section of `.looper.yaml` [#519](https://github.com/pepkit/looper/issues/519)
+- improve error messaging for bad/malformed looper configurations [#515](https://github.com/pepkit/looper/issues/515)
+- add shortform argument for --package (alias is now -p)
+
+## [1.9.1] -- 2024-07-18
 
 ### Changed
-- refactored some CLI arguments  [455](https://github.com/pepkit/looper/issues/455)
-- consolidate sample and project interfaces under a single pipeline interface [493](https://github.com/pepkit/looper/issues/493)
-- added guided tutorial for initializing looper config file [466](https://github.com/pepkit/looper/issues/466)
+- ensure peppy requirement peppy>=0.40.0,<=0.40.2
 
 ## [1.9.0] -- 2024-06-26
 
diff --git a/looper/_version.py b/looper/_version.py
index 33ca05a0..05c48b5c 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1,2 +1,2 @@
-__version__ = "2.0.0a1"
+__version__ = "2.0.0a2"
 # You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index df888fb4..9c66f794 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -340,7 +340,7 @@ def main(test_args=None) -> dict:
         prog="looper",
         description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="2.0.0a1",
+        version="2.0.0a2",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)

From 6a66d3100c9761b4e94449aba0ea7c42ce0e23cb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:28:38 -0400
Subject: [PATCH 47/56] refactor tests for new hello_looper changes

---
 tests/conftest.py                             | 10 ++---
 tests/data/hello_looper-dev/.looper.yaml      |  4 --
 tests/data/hello_looper-dev/README.md         | 12 +++--
 tests/data/hello_looper-dev/csv/.looper.yaml  |  5 ---
 .../hello_looper-dev/csv/data/frog1_data.txt  |  4 --
 .../hello_looper-dev/csv/data/frog2_data.txt  |  7 ---
 .../csv/pipeline/pipeline_interface.yaml      | 10 -----
 .../csv/project/sample_annotation.csv         |  3 --
 .../input_schema_example/.looper.yaml         |  4 ++
 .../input_schema_example/data/canada.txt      | 10 +++++
 .../input_schema_example/data/mexico.txt      | 31 +++++++++++++
 .../input_schema_example/data/switzerland.txt | 23 ++++++++++
 .../metadata/pep_config.yaml                  |  9 ++++
 .../metadata/sample_table.csv                 |  5 +++
 .../pipeline/count_lines.sh                   |  4 ++
 .../pipeline/input_schema.yaml                | 26 +++++++++++
 .../pipeline/pipeline_interface.yaml          | 10 +++++
 .../pipeline/resources-sample.tsv             |  4 ++
 .../looper_csv_example/.looper.yaml           |  4 ++
 .../looper_csv_example/data/canada.txt        | 10 +++++
 .../looper_csv_example/data/mexico.txt        | 31 +++++++++++++
 .../looper_csv_example/data/switzerland.txt   | 23 ++++++++++
 .../metadata/sample_table.csv                 |  4 ++
 .../pipeline/count_lines.sh                   |  2 +-
 .../pipeline/pipeline_interface.yaml          |  4 ++
 .../hello_looper-dev/minimal/.looper.yaml     |  5 ---
 .../minimal/pipeline/pipeline_interface.yaml  |  6 ---
 .../minimal/project/project_config.yaml       |  2 -
 .../minimal/project/sample_annotation.csv     |  3 --
 .../pep_derived_attrs/.looper.yaml            |  4 ++
 .../pep_derived_attrs/data/canada.txt         | 10 +++++
 .../pep_derived_attrs/data/mexico.txt         | 31 +++++++++++++
 .../pep_derived_attrs/data/switzerland.txt    | 23 ++++++++++
 .../metadata/pep_config.yaml                  |  9 ++++
 .../metadata/sample_table.csv                 |  4 ++
 .../pipeline/count_lines.sh                   |  2 +-
 .../pipeline/count_lines_plot.py              | 32 +++++++++++++
 .../pipeline/pipeline_interface.yaml          |  7 +++
 .../data/hello_looper-dev/pephub/.looper.yaml |  2 +-
 .../hello_looper-dev/pephub/data/canada.txt   | 10 +++++
 .../pephub/data/frog1_data.txt                |  4 --
 .../pephub/data/frog2_data.txt                |  7 ---
 .../hello_looper-dev/pephub/data/mexico.txt   | 31 +++++++++++++
 .../pephub/data/switzerland.txt               | 23 ++++++++++
 .../pephub/pipeline/pipeline_interface.yaml   |  8 +---
 .../hello_looper-dev/pipestat/data/frog_1.txt |  4 --
 .../hello_looper-dev/pipestat/data/frog_2.txt |  7 ---
 .../pipestat_example/.looper.yaml             |  8 ++++
 .../.looper_pipestat_shell.yaml               |  7 +++
 .../pipestat_example/data/canada.txt          | 10 +++++
 .../pipestat_example/data/mexico.txt          | 31 +++++++++++++
 .../pipestat_example/data/switzerland.txt     | 23 ++++++++++
 .../pipestat_example/metadata/pep_config.yaml |  9 ++++
 .../metadata/sample_table.csv                 |  4 ++
 .../pipeline}/count_lines.py                  |  0
 .../pipeline/count_lines.sh}                  |  0
 .../pipeline/count_lines_plot.py              | 45 +++++++++++++++++++
 .../pipeline/pipeline_interface.yaml          |  8 ++++
 .../pipeline/pipeline_interface_shell.yaml    |  5 +++
 .../pipeline/pipestat_output_schema.yaml      | 32 +++++++++++++
 .../data/hello_looper-dev/pytesting/README.md |  1 +
 .../advanced_test}/.looper.yaml               |  0
 .../.looper_advanced_pipestat.yaml            |  0
 .../advanced_test}/pipeline/col_pipeline1.py  |  0
 .../advanced_test}/pipeline/col_pipeline2.py  |  0
 .../pipeline/other_pipeline2.py               |  0
 .../pipeline/output_schema.yaml               |  0
 .../advanced_test}/pipeline/pipeline1.py      |  0
 .../pipeline/pipeline_interface1_project.yaml |  0
 .../pipeline/pipeline_interface1_sample.yaml  |  0
 .../pipeline/pipeline_interface2_project.yaml |  0
 .../pipeline/pipeline_interface2_sample.yaml  |  0
 .../pipeline/pipestat_output_schema.yaml      |  0
 .../pipestat_pipeline_interface1_sample.yaml  |  0
 .../pipestat_pipeline_interface2_sample.yaml  |  0
 .../advanced_test}/pipeline/readData.R        |  0
 .../pipeline/resources-project.tsv            |  0
 .../pipeline/resources-sample.tsv             |  0
 .../project/annotation_sheet.csv              |  0
 .../project/project_config.yaml               |  0
 .../intermediate_test}/.looper.yaml           |  0
 .../intermediate_test}/data/frog_1.txt        |  0
 .../intermediate_test}/data/frog_2.txt        |  0
 .../pipeline/count_lines.sh                   |  0
 .../pipeline/pipeline_interface.yaml          |  0
 .../project/project_config.yaml               |  0
 .../project/sample_annotation.csv             |  0
 .../pipestat_test}/.looper.yaml               |  0
 .../.looper_pipestat_shell.yaml               |  0
 .../pipestat_test}/data/frog_1.txt            |  0
 .../pipestat_test}/data/frog_2.txt            |  0
 .../pipeline_pipestat/count_lines.py          | 31 +++++++++++++
 .../pipeline_pipestat/count_lines_pipestat.sh |  4 ++
 .../pipeline_pipestat/pipeline_interface.yaml |  0
 .../pipeline_interface_shell.yaml             |  0
 .../pipestat_output_schema.yaml               |  0
 .../project/project_config.yaml               |  0
 .../project/sample_annotation.csv             |  0
 tests/smoketests/test_run.py                  |  2 +-
 99 files changed, 619 insertions(+), 94 deletions(-)
 delete mode 100644 tests/data/hello_looper-dev/.looper.yaml
 delete mode 100644 tests/data/hello_looper-dev/csv/.looper.yaml
 delete mode 100644 tests/data/hello_looper-dev/csv/data/frog1_data.txt
 delete mode 100644 tests/data/hello_looper-dev/csv/data/frog2_data.txt
 delete mode 100644 tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
 delete mode 100644 tests/data/hello_looper-dev/csv/project/sample_annotation.csv
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/canada.txt
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/mexico.txt
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv
 create mode 100755 tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/canada.txt
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv
 rename tests/data/hello_looper-dev/{intermediate => looper_csv_example}/pipeline/count_lines.sh (69%)
 create mode 100644 tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml
 delete mode 100644 tests/data/hello_looper-dev/minimal/.looper.yaml
 delete mode 100644 tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
 delete mode 100644 tests/data/hello_looper-dev/minimal/project/project_config.yaml
 delete mode 100644 tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv
 rename tests/data/hello_looper-dev/{minimal => pep_derived_attrs}/pipeline/count_lines.sh (69%)
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py
 create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev/pephub/data/canada.txt
 delete mode 100644 tests/data/hello_looper-dev/pephub/data/frog1_data.txt
 delete mode 100644 tests/data/hello_looper-dev/pephub/data/frog2_data.txt
 create mode 100644 tests/data/hello_looper-dev/pephub/data/mexico.txt
 create mode 100644 tests/data/hello_looper-dev/pephub/data/switzerland.txt
 delete mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_1.txt
 delete mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_2.txt
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/canada.txt
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/mexico.txt
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv
 rename tests/data/hello_looper-dev/{pipestat/pipeline_pipestat => pipestat_example/pipeline}/count_lines.py (100%)
 rename tests/data/hello_looper-dev/{pipestat/pipeline_pipestat/count_lines_pipestat.sh => pipestat_example/pipeline/count_lines.sh} (100%)
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml
 create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
 create mode 100644 tests/data/hello_looper-dev/pytesting/README.md
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/.looper.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/.looper_advanced_pipestat.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/col_pipeline1.py (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/col_pipeline2.py (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/other_pipeline2.py (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/output_schema.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline1.py (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface1_project.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface1_sample.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface2_project.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface2_sample.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_output_schema.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_pipeline_interface1_sample.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_pipeline_interface2_sample.yaml (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/readData.R (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/resources-project.tsv (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/resources-sample.tsv (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/project/annotation_sheet.csv (100%)
 rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/project/project_config.yaml (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/.looper.yaml (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/data/frog_1.txt (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/data/frog_2.txt (100%)
 rename tests/data/hello_looper-dev/{csv => pytesting/intermediate_test}/pipeline/count_lines.sh (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/pipeline/pipeline_interface.yaml (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/project/project_config.yaml (100%)
 rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/project/sample_annotation.csv (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/.looper.yaml (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/.looper_pipestat_shell.yaml (100%)
 rename tests/data/hello_looper-dev/{minimal => pytesting/pipestat_test}/data/frog_1.txt (100%)
 rename tests/data/hello_looper-dev/{minimal => pytesting/pipestat_test}/data/frog_2.txt (100%)
 create mode 100755 tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py
 create mode 100755 tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipeline_interface.yaml (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipeline_interface_shell.yaml (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipestat_output_schema.yaml (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/project/project_config.yaml (100%)
 rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/project/sample_annotation.csv (100%)

diff --git a/tests/conftest.py b/tests/conftest.py
index cfc10457..960a98b4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -204,7 +204,7 @@ def prep_temp_pep(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "advanced")
+    advanced_dir = os.path.join(d, "pytesting/advanced_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -220,7 +220,7 @@ def prep_temp_pep_basic(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "intermediate")
+    advanced_dir = os.path.join(d, "pytesting/intermediate_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -236,7 +236,7 @@ def prep_temp_pep_csv(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "csv")
+    advanced_dir = os.path.join(d, "looper_csv_example")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -274,7 +274,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "pipestat")
+    advanced_dir = os.path.join(d, "pytesting/pipestat_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -291,7 +291,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "advanced")
+    advanced_dir = os.path.join(d, "pytesting/advanced_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml")
 
     return path_to_looper_config
diff --git a/tests/data/hello_looper-dev/.looper.yaml b/tests/data/hello_looper-dev/.looper.yaml
deleted file mode 100644
index e812a1ea..00000000
--- a/tests/data/hello_looper-dev/.looper.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-pep_config: ./project/project_config.yaml # pephub registry path or local path
-output_dir: "./results"
-pipeline_interfaces:
-  sample:  ../pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/README.md b/tests/data/hello_looper-dev/README.md
index 6c213b1a..a2a75fee 100644
--- a/tests/data/hello_looper-dev/README.md
+++ b/tests/data/hello_looper-dev/README.md
@@ -4,12 +4,10 @@ This repository provides minimal working examples for the [looper pipeline submi
 
 This repository contains examples
 
-1. `/minimal` - A basic example pipeline and project.
-2. `/intermediate` - An intermediate example pipeline and project with a couple extra options.
-3. `/advanced` - A more advanced example, showcasing the capabilities of Looper.
-4. `/pephub` - Example of how to point looper to PEPhub.
-5. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
-6. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
+1. `/looper_csv_example` - A minimal example using _only_ csv for metadata.
+2. `/pep_derived_attributes` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata
+3. `/pephub` - Example of how to point looper to a PEP stored on PEPhub and running a pipeline.
+4. `/pipestat` - Example on how to use pipestat to report pipeline results when using looper.
 
 Each example contains:
 
@@ -17,4 +15,4 @@ Each example contains:
 2. Sample data plus metadata in PEP format (or pointer to PEPhub).
 3. A looper-compatible pipeline.
 
-Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/)
+Explanation and results of running the above examples can be found at [Looper: Hello World Tutorial](https://pep.databio.org/looper/tutorial/initialize/)
diff --git a/tests/data/hello_looper-dev/csv/.looper.yaml b/tests/data/hello_looper-dev/csv/.looper.yaml
deleted file mode 100644
index 886b301c..00000000
--- a/tests/data/hello_looper-dev/csv/.looper.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pep_config: project/sample_annotation.csv # local path to CSV
-# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
-output_dir: "results"
-pipeline_interfaces:
-  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/csv/data/frog1_data.txt b/tests/data/hello_looper-dev/csv/data/frog1_data.txt
deleted file mode 100644
index 815c0cf7..00000000
--- a/tests/data/hello_looper-dev/csv/data/frog1_data.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-ribbit
-ribbit
-ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/csv/data/frog2_data.txt b/tests/data/hello_looper-dev/csv/data/frog2_data.txt
deleted file mode 100644
index e6fdd535..00000000
--- a/tests/data/hello_looper-dev/csv/data/frog2_data.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-ribbit
-ribbit
-ribbit
-
-ribbit, ribbit
-ribbit, ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
deleted file mode 100644
index 1b8a77eb..00000000
--- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-pipeline_name: count_lines
-pipeline_type: sample
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-sample_interface:
-  command_template: >
-    {pipeline.var_templates.pipeline} {sample.file}
-project_interface:
-  command_template: >
-    {pipeline.var_templates.pipeline} "data/*.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv b/tests/data/hello_looper-dev/csv/project/sample_annotation.csv
deleted file mode 100644
index 05bf4d17..00000000
--- a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-sample_name,library,file,toggle
-frog_1,anySampleType,data/frog1_data.txt,1
-frog_2,anySampleType,data/frog2_data.txt,1
diff --git a/tests/data/hello_looper-dev/input_schema_example/.looper.yaml b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml
new file mode 100644
index 00000000..ba4eb61d
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml
@@ -0,0 +1,4 @@
+pep_config: metadata/pep_config.yaml
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/input_schema_example/data/canada.txt b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt
new file mode 100644
index 00000000..dd70801c
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt
@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan
diff --git a/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt
new file mode 100644
index 00000000..66cf49d5
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt
@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt
new file mode 100644
index 00000000..b7159d55
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt
@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura
diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml
new file mode 100644
index 00000000..45cefe21
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml
@@ -0,0 +1,9 @@
+pep_version: 2.1.0
+sample_table: sample_table.csv
+sample_modifiers:
+  append:
+    file_path: source1
+  derive:
+    attributes: [file_path]
+    sources:
+      source1: "data/{sample_name}.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv
new file mode 100644
index 00000000..dcbd2366
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv
@@ -0,0 +1,5 @@
+sample_name,area_type
+mexico,state
+switzerland,canton
+canada,province
+usa,
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh
new file mode 100755
index 00000000..03675d99
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+export area_type=$2
+echo "Number of ${area_type}s: $linecount"
diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml
new file mode 100644
index 00000000..1de22ea3
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml
@@ -0,0 +1,26 @@
+description: An input schema for count_lines pipeline pipeline.
+properties:
+  samples:
+    type: array
+    items:
+      type: object
+      properties:
+        sample_name: 
+          type: string
+          description: "Name of the sample"
+        file_path: 
+          type: string
+          description: "Path to the input file to count"
+        area_type:
+          type: string
+          description: "Name of the components of the country"
+      tangible:
+        - file_path
+      sizing:
+        - file_path
+      required:
+        - sample_name
+        - area_type
+        - file_path
+required:
+  - samples
diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml
new file mode 100644
index 00000000..6f7b156b
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml
@@ -0,0 +1,10 @@
+pipeline_name: count_lines
+input_schema: input_schema.yaml
+sample_interface:
+  command_template: >
+    pipeline/count_lines.sh {sample.file_path} {sample.area_type}
+project_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/
+compute:
+  size_dependent_variables: resources-sample.tsv
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv
new file mode 100644
index 00000000..6f0553ef
--- /dev/null
+++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv
@@ -0,0 +1,4 @@
+max_file_size	cores	mem	time
+0.0005	1	1000	00-01:00:00
+0.05	2	2000	00-03:00:00
+NaN	4	4000	00-05:00:00
diff --git a/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml
new file mode 100644
index 00000000..381d1819
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml
@@ -0,0 +1,4 @@
+pep_config: metadata/sample_table.csv
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt
new file mode 100644
index 00000000..dd70801c
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt
@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan
diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt
new file mode 100644
index 00000000..66cf49d5
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt
@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt
new file mode 100644
index 00000000..b7159d55
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt
@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura
diff --git a/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv
new file mode 100644
index 00000000..6aa56d5b
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv
@@ -0,0 +1,4 @@
+sample_name,area_type,file_path
+mexico,state,data/mexico.txt
+switzerland,canton,data/switzerland.txt
+canada,province,data/canada.txt
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh
similarity index 69%
rename from tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh
index 71b887fe..0a39a4a8 100755
--- a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh
+++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
-echo "Number of lines: $linecount"
+echo "Number of lines: $linecount"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml
new file mode 100644
index 00000000..dc58569b
--- /dev/null
+++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml
@@ -0,0 +1,4 @@
+pipeline_name: count_lines
+sample_interface:
+  command_template: >
+    pipeline/count_lines.sh {sample.file_path}
diff --git a/tests/data/hello_looper-dev/minimal/.looper.yaml b/tests/data/hello_looper-dev/minimal/.looper.yaml
deleted file mode 100644
index 4fcf5672..00000000
--- a/tests/data/hello_looper-dev/minimal/.looper.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pep_config: project/project_config.yaml # local path to pep config
-# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
-output_dir: "results"
-pipeline_interfaces:
-  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
deleted file mode 100644
index 58ddbf2b..00000000
--- a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pipeline_name: count_lines
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
-sample_interface:
-  command_template: >
-    {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev/minimal/project/project_config.yaml b/tests/data/hello_looper-dev/minimal/project/project_config.yaml
deleted file mode 100644
index 5456cca3..00000000
--- a/tests/data/hello_looper-dev/minimal/project/project_config.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-pep_version: 2.0.0
-sample_table: sample_annotation.csv
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv b/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
deleted file mode 100644
index 97f22370..00000000
--- a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-sample_name,library,file,toggle
-frog_1,anySampleType,data/frog_1.txt,1
-frog_2,anySampleType,data/frog_2.txt,1
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml
new file mode 100644
index 00000000..ba4eb61d
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml
@@ -0,0 +1,4 @@
+pep_config: metadata/pep_config.yaml
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt
new file mode 100644
index 00000000..dd70801c
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt
@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt
new file mode 100644
index 00000000..66cf49d5
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt
@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt
new file mode 100644
index 00000000..b7159d55
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt
@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml
new file mode 100644
index 00000000..45cefe21
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml
@@ -0,0 +1,9 @@
+pep_version: 2.1.0
+sample_table: sample_table.csv
+sample_modifiers:
+  append:
+    file_path: source1
+  derive:
+    attributes: [file_path]
+    sources:
+      source1: "data/{sample_name}.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv
new file mode 100644
index 00000000..85f38a90
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv
@@ -0,0 +1,4 @@
+sample_name,area_type
+mexico,state
+switzerland,canton
+canada,province
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh
similarity index 69%
rename from tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh
index 71b887fe..0a39a4a8 100755
--- a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
-echo "Number of lines: $linecount"
+echo "Number of lines: $linecount"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py
new file mode 100644
index 00000000..398c1c02
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py
@@ -0,0 +1,32 @@
+import matplotlib.pyplot as plt
+import os
+import sys
+
+results_dir = sys.argv[
+    1
+]  # Obtain the looper results directory passed via the looper command template
+
+# Extract the previously reported sample-level data from the .log files
+countries = []
+number_of_regions = []
+for filename in os.listdir(results_dir):
+    if filename.endswith(".log"):
+        file = os.path.join(results_dir, filename)
+        with open(file, "r") as f:
+            for line in f:
+                if line.startswith("Number of lines:"):
+                    region_count = int(line.split(":")[1].strip())
+                    number_of_regions.append(region_count)
+                    country = filename.split("_")[2].split(".")[0]
+                    countries.append(country)
+
+# Create a bar chart of regions per country
+plt.figure(figsize=(8, 5))
+plt.bar(countries, number_of_regions, color=["blue", "green", "purple"])
+plt.xlabel("Countries")
+plt.ylabel("Number of regions")
+plt.title("Number of regions per country")
+
+# Save the image locally
+save_location = os.path.join(os.path.dirname(results_dir), "regions_per_country.png")
+plt.savefig(save_location, dpi=150)
diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml
new file mode 100644
index 00000000..dacc6589
--- /dev/null
+++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml
@@ -0,0 +1,7 @@
+pipeline_name: count_lines
+sample_interface:
+  command_template: >
+    pipeline/count_lines.sh {sample.file_path}
+project_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/
diff --git a/tests/data/hello_looper-dev/pephub/.looper.yaml b/tests/data/hello_looper-dev/pephub/.looper.yaml
index c2d74e0c..654c5427 100644
--- a/tests/data/hello_looper-dev/pephub/.looper.yaml
+++ b/tests/data/hello_looper-dev/pephub/.looper.yaml
@@ -1,4 +1,4 @@
-pep_config: pepkit/hello_looper:default # pephub registry path or local path
+pep_config: databio/hello_looper:default 
 output_dir: results
 pipeline_interfaces:
   - pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/pephub/data/canada.txt b/tests/data/hello_looper-dev/pephub/data/canada.txt
new file mode 100644
index 00000000..dd70801c
--- /dev/null
+++ b/tests/data/hello_looper-dev/pephub/data/canada.txt
@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan
diff --git a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt b/tests/data/hello_looper-dev/pephub/data/frog1_data.txt
deleted file mode 100644
index 815c0cf7..00000000
--- a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-ribbit
-ribbit
-ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt b/tests/data/hello_looper-dev/pephub/data/frog2_data.txt
deleted file mode 100644
index e6fdd535..00000000
--- a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-ribbit
-ribbit
-ribbit
-
-ribbit, ribbit
-ribbit, ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/pephub/data/mexico.txt b/tests/data/hello_looper-dev/pephub/data/mexico.txt
new file mode 100644
index 00000000..66cf49d5
--- /dev/null
+++ b/tests/data/hello_looper-dev/pephub/data/mexico.txt
@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pephub/data/switzerland.txt b/tests/data/hello_looper-dev/pephub/data/switzerland.txt
new file mode 100644
index 00000000..b7159d55
--- /dev/null
+++ b/tests/data/hello_looper-dev/pephub/data/switzerland.txt
@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura
diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
index 9bea7b43..dc58569b 100644
--- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
+++ b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
@@ -1,10 +1,4 @@
 pipeline_name: count_lines
-pipeline_type: sample
-var_templates:
-  pipeline: '{looper.piface_dir}/count_lines.sh'
 sample_interface:
   command_template: >
-    {pipeline.var_templates.pipeline} {sample.file}
-project_interface:
-  command_template: >
-    {pipeline.var_templates.pipeline} "data/*.txt"
+    pipeline/count_lines.sh {sample.file_path}
diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev/pipestat/data/frog_1.txt
deleted file mode 100644
index 815c0cf7..00000000
--- a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-ribbit
-ribbit
-ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev/pipestat/data/frog_2.txt
deleted file mode 100644
index e6fdd535..00000000
--- a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-ribbit
-ribbit
-ribbit
-
-ribbit, ribbit
-ribbit, ribbit
-CROAK!
diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml
new file mode 100644
index 00000000..dba1fbe9
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml
@@ -0,0 +1,8 @@
+pep_config: ./metadata/pep_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml
+pipestat:
+  project_name: count_lines
+  results_file_path: results.yaml
+  flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml
new file mode 100644
index 00000000..29a39719
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml
@@ -0,0 +1,7 @@
+pep_config: ./metadata/pep_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  - pipeline/pipeline_interface_shell.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/data/canada.txt b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt
new file mode 100644
index 00000000..dd70801c
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt
@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan
diff --git a/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt
new file mode 100644
index 00000000..66cf49d5
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt
@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt
new file mode 100644
index 00000000..b7159d55
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt
@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura
diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml
new file mode 100644
index 00000000..45cefe21
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml
@@ -0,0 +1,9 @@
+pep_version: 2.1.0
+sample_table: sample_table.csv
+sample_modifiers:
+  append:
+    file_path: source1
+  derive:
+    attributes: [file_path]
+    sources:
+      source1: "data/{sample_name}.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv
new file mode 100644
index 00000000..85f38a90
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv
@@ -0,0 +1,4 @@
+sample_name,area_type
+mexico,state
+switzerland,canton
+canada,province
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py
rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh
rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh
diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py
new file mode 100644
index 00000000..bc3a2bce
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py
@@ -0,0 +1,45 @@
+import matplotlib.pyplot as plt  # be sure to `pip install matplotlib`
+import os
+import pipestat
+import sys
+
+# A pipeline that retrieves previously reported pipestat results
+# and plots them in a bar chart
+results_file = sys.argv[1]
+schema_path = sys.argv[2]
+
+# Create pipestat manager
+psm = pipestat.PipestatManager(
+    schema_path=schema_path, results_file_path=results_file, pipeline_type="project"
+)
+
+# Extract the previously reported data
+results = (
+    psm.select_records()
+)  # pipestat object holds the data after reading the results file
+countries = [record["record_identifier"] for record in results["records"]]
+number_of_regions = [record["number_of_lines"] for record in results["records"]]
+
+# Create a bar chart of regions per country
+plt.figure(figsize=(8, 5))
+plt.bar(countries, number_of_regions, color=["blue", "green", "purple"])
+plt.xlabel("Countries")
+plt.ylabel("Number of regions")
+plt.title("Number of regions per country")
+# plt.show() # Showing the figure and then saving it causes issues, so leave this commented out.
+
+# Save the image locally AND report that location via pipestat
+# we can place it next to the results file for now
+save_location = os.path.join(os.path.dirname(results_file), "regions_per_country.png")
+
+plt.savefig(save_location, dpi=150)
+
+result_to_report = {
+    "regions_plot": {
+        "path": save_location,
+        "thumbnail_path": save_location,
+        "title": "Regions Plot",
+    }
+}
+
+psm.report(record_identifier="count_lines", values=result_to_report)
diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml
new file mode 100644
index 00000000..275af085
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml
@@ -0,0 +1,8 @@
+pipeline_name: count_lines
+output_schema: pipestat_output_schema.yaml
+sample_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines.py {sample.file_path} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema}
+project_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines_plot.py {pipestat.results_file} {pipestat.output_schema}
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml
new file mode 100644
index 00000000..c3b930fb
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml
@@ -0,0 +1,5 @@
+pipeline_name: count_lines
+output_schema: pipestat_output_schema.yaml
+sample_interface:
+  command_template: >
+    {looper.piface_dir}/count_lines.sh {sample.file_path} {sample.sample_name} {pipestat.config_file}
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
new file mode 100644
index 00000000..7bedfd7e
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
@@ -0,0 +1,32 @@
+title: Pipestat output schema for counting lines
+description: A pipeline that uses pipestat to report sample level results.
+type: object
+properties:
+  pipeline_name: count_lines
+  samples:
+    type: array
+    items:
+      type: object
+      properties:
+        number_of_lines:
+          type: integer
+          description: "Number of lines in the input file."
+  project:
+    type: object
+    properties:
+      regions_plot:
+        description: "This a path to the output image"
+        image:
+          type: object
+          object_type: image
+          properties:
+            path:
+              type: string
+            thumbnail_path:
+              type: string
+            title:
+              type: string
+          required:
+            - path
+            - thumbnail_path
+            - title
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/pytesting/README.md b/tests/data/hello_looper-dev/pytesting/README.md
new file mode 100644
index 00000000..1f0c5cc7
--- /dev/null
+++ b/tests/data/hello_looper-dev/pytesting/README.md
@@ -0,0 +1 @@
+The examples in this pytesting directory are for Looper testing purposes.
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/advanced/.looper.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/.looper.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml
diff --git a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/readData.R b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/readData.R
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv
diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv
rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv
diff --git a/tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv b/tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv
rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv
diff --git a/tests/data/hello_looper-dev/advanced/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/advanced/project/project_config.yaml
rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/.looper.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/.looper.yaml
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/data/frog_1.txt
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt
diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/data/frog_2.txt
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt
diff --git a/tests/data/hello_looper-dev/csv/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh
similarity index 100%
rename from tests/data/hello_looper-dev/csv/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh
diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/project/project_config.yaml
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv
similarity index 100%
rename from tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv
rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv
diff --git a/tests/data/hello_looper-dev/pipestat/.looper.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/.looper.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml
diff --git a/tests/data/hello_looper-dev/minimal/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt
similarity index 100%
rename from tests/data/hello_looper-dev/minimal/data/frog_1.txt
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt
diff --git a/tests/data/hello_looper-dev/minimal/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt
similarity index 100%
rename from tests/data/hello_looper-dev/minimal/data/frog_2.txt
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt
diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py
new file mode 100755
index 00000000..6f6a4ab8
--- /dev/null
+++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py
@@ -0,0 +1,31 @@
+import os.path
+
+import pipestat
+import sys
+
+# Very simple pipeline that calls pipestat
+# takes arguments invoked during looper submission via command templates
+text_file = sys.argv[
+    1
+]  # this is the sample we wish to process by reading the number of lines
+sample_name = sys.argv[2]
+results_file = sys.argv[3]
+schema_path = sys.argv[4]
+
+# Create pipestat manager and then report values
+psm = pipestat.PipestatManager(
+    schema_path=schema_path,
+    results_file_path=results_file,
+    record_identifier=sample_name,
+)
+
+
+text_file = os.path.abspath(text_file)
+# Read text file and count lines
+with open(text_file, "r") as f:
+    result = {"number_of_lines": len(f.readlines())}
+
+# The results are defined in the pipestat output schema.
+psm.report(record_identifier=sample_name, values=result)
+
+# end of pipeline
diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh
new file mode 100755
index 00000000..99f83f90
--- /dev/null
+++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/project/project_config.yaml
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv
similarity index 100%
rename from tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv
rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 59eede55..c35d5947 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -634,4 +634,4 @@ def test_init_project_using_csv(self, prep_temp_pep_csv):
         pep_config_csv = os.path.join(os.path.dirname(tp), pep_config_csv)
         init_project = Project(cfg=pep_config_csv)
 
-        assert len(init_project.samples) == 2
+        assert len(init_project.samples) == 3

From 879664605115ebcdb8b53b52a5bd1b53b0c5f67e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:37:37 -0400
Subject: [PATCH 48/56] pull updated hello_looper readme

---
 tests/data/hello_looper-dev/README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/data/hello_looper-dev/README.md b/tests/data/hello_looper-dev/README.md
index a2a75fee..fc4b8692 100644
--- a/tests/data/hello_looper-dev/README.md
+++ b/tests/data/hello_looper-dev/README.md
@@ -5,9 +5,10 @@ This repository provides minimal working examples for the [looper pipeline submi
 This repository contains examples
 
 1. `/looper_csv_example` - A minimal example using _only_ csv for metadata.
-2. `/pep_derived_attributes` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata
+2. `/pep_derived_attrs` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata
 3. `/pephub` - Example of how to point looper to a PEP stored on PEPhub and running a pipeline.
-4. `/pipestat` - Example on how to use pipestat to report pipeline results when using looper.
+4. `/pipestat_example` - Example on how to use pipestat to report pipeline results when using looper.
+5. `/input_schema_example` - Example on how to use input schemas when using looper.
 
 Each example contains:
 
@@ -15,4 +16,4 @@ Each example contains:
 2. Sample data plus metadata in PEP format (or pointer to PEPhub).
 3. A looper-compatible pipeline.
 
-Explanation and results of running the above examples can be found at [Looper: Hello World Tutorial](https://pep.databio.org/looper/tutorial/initialize/)
+Explanation and results of running the above examples can be found at [Looper: User Tutorial](https://pep.databio.org/looper/user-tutorial/initialize/)

From b54e678991b9c08d2201a5d1b06272f247013837 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 1 Oct 2024 16:53:42 -0400
Subject: [PATCH 49/56] pull updated hello_looper output schema

---
 .../pipestat_example/pipeline/pipestat_output_schema.yaml        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
index 7bedfd7e..358b1f77 100644
--- a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
@@ -15,6 +15,7 @@ properties:
     type: object
     properties:
       regions_plot:
+        type: object
         description: "This a path to the output image"
         image:
           type: object

From 89cbfc2ed983e1ce60dd623ac34f796882b8c52d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 2 Oct 2024 11:32:58 -0400
Subject: [PATCH 50/56] pull updated hello_looper output schema (again)

---
 .../pipeline/pipestat_output_schema.yaml      | 28 +++++++++----------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
index 358b1f77..be8007a5 100644
--- a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
+++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml
@@ -15,19 +15,17 @@ properties:
     type: object
     properties:
       regions_plot:
-        type: object
         description: "This a path to the output image"
-        image:
-          type: object
-          object_type: image
-          properties:
-            path:
-              type: string
-            thumbnail_path:
-              type: string
-            title:
-              type: string
-          required:
-            - path
-            - thumbnail_path
-            - title
\ No newline at end of file
+        type: object
+        object_type: image
+        properties:
+          path:
+            type: string
+          thumbnail_path:
+            type: string
+          title:
+            type: string
+        required:
+          - path
+          - thumbnail_path
+          - title
\ No newline at end of file

From e0aaa04f070eede61e1bca1fb34a5cb05fd8110b Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Wed, 9 Oct 2024 07:47:16 -0400
Subject: [PATCH 51/56] use filepath (requires new yacman)

---
 looper/divvy.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 856219cc..c9f97258 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -111,9 +111,12 @@ def templates_folder(self):
 
         :return str: path to folder with default submission templates
         """
-        return os.path.join(
-            os.path.dirname(__file__), "default_config", "divvy_templates"
-        )
+        if self.filepath:
+            return os.path.join(os.path.dirname(self.filepath), "divvy_templates")
+        else:
+            return os.path.join(
+                os.path.dirname(__file__), "default_config", "divvy_templates"
+            )
 
     def activate_package(self, package_name):
         """
@@ -155,11 +158,18 @@ def activate_package(self, package_name):
             # but now, it makes more sense to do it here so we can piggyback on
             # the default update() method and not even have to do that.
             if not os.path.isabs(self.compute["submission_template"]):
+                
                 try:
-                    self.compute["submission_template"] = os.path.join(
-                        os.path.dirname(self.default_config_file),
-                        self.compute["submission_template"],
-                    )
+                    if self.filepath:
+                        self.compute["submission_template"] = os.path.join(
+                            os.path.dirname(self.filepath),
+                            self.compute["submission_template"],
+                        )
+                    else:
+                        self.compute["submission_template"] = os.path.join(
+                            os.path.dirname(self.default_config_file),
+                            self.compute["submission_template"],
+                        )
                 except AttributeError as e:
                     # Environment and environment compute should at least have been
                     # set as null-valued attributes, so execution here is an error.

From 222abac8b55f294683b8a2e0f048f4aec48697d1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 20 Dec 2024 15:30:57 -0500
Subject: [PATCH 52/56] solve #537

---
 looper/exceptions.py              |  7 +++++++
 looper/looper.py                  | 19 +++++++++++++------
 requirements/requirements-all.txt |  2 +-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/looper/exceptions.py b/looper/exceptions.py
index 469f68af..d0a4b5d2 100644
--- a/looper/exceptions.py
+++ b/looper/exceptions.py
@@ -15,6 +15,7 @@
     "PipelineInterfaceConfigError",
     "PipelineInterfaceRequirementsError",
     "MisconfigurationException",
+    "LooperReportError",
 ]
 
 
@@ -109,3 +110,9 @@ def __init__(self, typename_by_requirement):
             )
         )
         self.error_specs = typename_by_requirement
+
+class LooperReportError(LooperError):
+    """Looper reporting errors"""
+
+    def __init__(self, reason):
+        super(LooperReportError, self).__init__(reason)
\ No newline at end of file
diff --git a/looper/looper.py b/looper/looper.py
index 18f0d9ed..8a1bac05 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -46,6 +46,7 @@
     sample_folder,
 )
 from pipestat.reports import get_file_for_table
+from pipestat.exceptions import PipestatSummarizeError
 
 _PKGNAME = "looper"
 _LOGGER = logging.getLogger(_PKGNAME)
@@ -566,9 +567,12 @@ def __call__(self, args):
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
-                    report_directory = piface.psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
-                    )
+                    try:
+                        report_directory = piface.psm.summarize(
+                            looper_samples=self.prj.samples, portable=portable
+                        )
+                    except PipestatSummarizeError as e:
+                        raise LooperReportError(f"Looper report error due to the following exception: {e}")
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
@@ -576,9 +580,12 @@ def __call__(self, args):
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                     psms[piface.psm.pipeline_name] = piface.psm
-                    report_directory = piface.psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
-                    )
+                    try:
+                        report_directory = piface.psm.summarize(
+                            looper_samples=self.prj.samples, portable=portable
+                        )
+                    except PipestatSummarizeError as e:
+                        raise LooperReportError(f"Looper report error due to the following exception: {e}")
                     print(f"Report directory: {report_directory}")
                     self.debug["report_directory"] = report_directory
             return self.debug
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 5005921c..7351b913 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -4,7 +4,7 @@ jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
-pipestat>=0.10.2
+pipestat>=0.11.1a1
 peppy>=0.40.6
 pyyaml>=3.12
 rich>=9.10.0

From fed3fd30d558f6627b0384cd5f7adb7d2cc0c899 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 23 Dec 2024 10:00:40 -0500
Subject: [PATCH 53/56] solution for #536 and #522

---
 looper/looper.py               | 60 ++++++++++++++++++++--------------
 tests/smoketests/test_other.py | 20 ++++++++----
 tests/test_comprehensive.py    |  2 +-
 3 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 8a1bac05..a9d45d05 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -95,11 +95,15 @@ def __call__(self, args):
 
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    s = piface.psm.get_status() or "unknown"
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                all_project_level_records = psm.select_records()
+                for record in all_project_level_records['records']:
+                    s = piface.psm.get_status(record_identifier=record['record_identifier'])
                     status.setdefault(piface.psm.pipeline_name, {})
-                    status[piface.psm.pipeline_name][self.prj.name] = s
-                    _LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}")
+                    status[piface.psm.pipeline_name][record['record_identifier']] = s
+                    _LOGGER.debug(f"{self.prj.name} ({record['record_identifier']}): {s}")
 
         else:
             for sample in self.prj.samples:
@@ -566,28 +570,32 @@ def __call__(self, args):
 
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    try:
-                        report_directory = piface.psm.summarize(
-                            looper_samples=self.prj.samples, portable=portable
-                        )
-                    except PipestatSummarizeError as e:
-                        raise LooperReportError(f"Looper report error due to the following exception: {e}")
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                try:
+                    report_directory = psm.summarize(
+                        looper_samples=self.prj.samples, portable=portable
+                    )
+                except PipestatSummarizeError as e:
+                    raise LooperReportError(f"Looper report error due to the following exception: {e}")
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
         else:
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    try:
-                        report_directory = piface.psm.summarize(
-                            looper_samples=self.prj.samples, portable=portable
-                        )
-                    except PipestatSummarizeError as e:
-                        raise LooperReportError(f"Looper report error due to the following exception: {e}")
-                    print(f"Report directory: {report_directory}")
-                    self.debug["report_directory"] = report_directory
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                try:
+                    report_directory = psm.summarize(
+                        looper_samples=self.prj.samples, portable=portable
+                    )
+                except PipestatSummarizeError as e:
+                    raise LooperReportError(f"Looper report error due to the following exception: {e}")
+                print(f"Report directory: {report_directory}")
+                self.debug["report_directory"] = report_directory
             return self.debug
 
 
@@ -630,13 +638,17 @@ def __call__(self, args):
         if project_level:
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    results = piface.psm.table()
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                results = psm.table()
         else:
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    results = piface.psm.table()
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                results = psm.table()
         # Results contains paths to stats and object summaries.
         return results
 
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 9713d16a..85f65f3a 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -3,7 +3,7 @@
 import pytest
 from peppy import Project
 
-from looper.exceptions import PipestatConfigurationException, MisconfigurationException
+from looper.exceptions import PipestatConfigurationException, MisconfigurationException, LooperReportError
 from tests.conftest import *
 from looper.cli_pydantic import main
 import pandas as pd
@@ -78,12 +78,18 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
             # Not every command supports dry run
             x = [cmd, "--config", tp]
 
-        try:
-            result = main(test_args=x)
-            if cmd == "run":
-                assert result["Pipestat compatible"] is True
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        if cmd not in ["report"]:
+            try:
+                result = main(test_args=x)
+                if cmd == "run":
+                    assert result["Pipestat compatible"] is True
+            except Exception:
+                raise pytest.fail("DID RAISE {0}".format(Exception))
+        else:
+            with pytest.raises(expected_exception=LooperReportError): # Looper report will and should raise exception if there are no results reported.
+                result = main(test_args=x)
+
+
 
 
 class TestLooperRerun:
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index cb79b356..41c73ea0 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -123,7 +123,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     try:
         result = main(test_args=x)
-        assert result == {"example_pipestat_pipeline": {"project": "unknown"}}
+        assert result == {}
 
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))

From a7c5bd5a0f84a087a848a2789b782bc9f9cebac3 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 30 Dec 2024 13:23:06 -0500
Subject: [PATCH 54/56] add ability to specify path for report and table
 commands #534

---
 looper/command_models/arguments.py |  6 +++++
 looper/command_models/commands.py  |  5 ++++-
 looper/divvy.py                    |  2 +-
 looper/exceptions.py               |  3 ++-
 looper/looper.py                   | 35 +++++++++++++++++++++---------
 requirements/requirements-all.txt  |  2 +-
 tests/smoketests/test_other.py     | 12 ++++++----
 7 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 2d6b2fbc..68c32977 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -184,6 +184,12 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Output directory",
     )
+    REPORT_OUTPUT_DIR = Argument(
+        name="report_dir",
+        alias="-r",
+        default=(str, None),
+        description="Set location for looper report and looper table outputs",
+    )
 
     GENERIC = Argument(
         name="generic",
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 1df4662c..69312f0d 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -124,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 TableParser = Command(
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
-    [],
+    [
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
+    ],
 )
 
 
@@ -134,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
     ],
 )
 
diff --git a/looper/divvy.py b/looper/divvy.py
index c9f97258..e458bad6 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -158,7 +158,7 @@ def activate_package(self, package_name):
             # but now, it makes more sense to do it here so we can piggyback on
             # the default update() method and not even have to do that.
             if not os.path.isabs(self.compute["submission_template"]):
-                
+
                 try:
                     if self.filepath:
                         self.compute["submission_template"] = os.path.join(
diff --git a/looper/exceptions.py b/looper/exceptions.py
index d0a4b5d2..7d478feb 100644
--- a/looper/exceptions.py
+++ b/looper/exceptions.py
@@ -111,8 +111,9 @@ def __init__(self, typename_by_requirement):
         )
         self.error_specs = typename_by_requirement
 
+
 class LooperReportError(LooperError):
     """Looper reporting errors"""
 
     def __init__(self, reason):
-        super(LooperReportError, self).__init__(reason)
\ No newline at end of file
+        super(LooperReportError, self).__init__(reason)
diff --git a/looper/looper.py b/looper/looper.py
index a9d45d05..cb3cb301 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -99,11 +99,15 @@ def __call__(self, args):
                         psms[piface.psm.pipeline_name] = piface.psm
             for pl_name, psm in psms.items():
                 all_project_level_records = psm.select_records()
-                for record in all_project_level_records['records']:
-                    s = piface.psm.get_status(record_identifier=record['record_identifier'])
+                for record in all_project_level_records["records"]:
+                    s = piface.psm.get_status(
+                        record_identifier=record["record_identifier"]
+                    )
                     status.setdefault(piface.psm.pipeline_name, {})
-                    status[piface.psm.pipeline_name][record['record_identifier']] = s
-                    _LOGGER.debug(f"{self.prj.name} ({record['record_identifier']}): {s}")
+                    status[piface.psm.pipeline_name][record["record_identifier"]] = s
+                    _LOGGER.debug(
+                        f"{self.prj.name} ({record['record_identifier']}): {s}"
+                    )
 
         else:
             for sample in self.prj.samples:
@@ -564,6 +568,8 @@ def __call__(self, args):
 
         portable = args.portable
 
+        report_dir = getattr(args, "report_dir", None)
+
         psms = {}
 
         if project_level:
@@ -575,10 +581,14 @@ def __call__(self, args):
             for pl_name, psm in psms.items():
                 try:
                     report_directory = psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
+                        looper_samples=self.prj.samples,
+                        portable=portable,
+                        output_dir=report_dir,
                     )
                 except PipestatSummarizeError as e:
-                    raise LooperReportError(f"Looper report error due to the following exception: {e}")
+                    raise LooperReportError(
+                        f"Looper report error due to the following exception: {e}"
+                    )
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
@@ -590,10 +600,14 @@ def __call__(self, args):
             for pl_name, psm in psms.items():
                 try:
                     report_directory = psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
+                        looper_samples=self.prj.samples,
+                        portable=portable,
+                        output_dir=report_dir,
                     )
                 except PipestatSummarizeError as e:
-                    raise LooperReportError(f"Looper report error due to the following exception: {e}")
+                    raise LooperReportError(
+                        f"Looper report error due to the following exception: {e}"
+                    )
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
@@ -633,6 +647,7 @@ class Tabulator(Executor):
     def __call__(self, args):
         # p = self.prj
         project_level = getattr(args, "project", None)
+        report_dir = getattr(args, "report_dir", None)
         results = []
         psms = {}
         if project_level:
@@ -641,14 +656,14 @@ def __call__(self, args):
                     if piface.psm.pipeline_name not in psms:
                         psms[piface.psm.pipeline_name] = piface.psm
             for pl_name, psm in psms.items():
-                results = psm.table()
+                results = psm.table(output_dir=report_dir)
         else:
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                     if piface.psm.pipeline_name not in psms:
                         psms[piface.psm.pipeline_name] = piface.psm
             for pl_name, psm in psms.items():
-                results = psm.table()
+                results = psm.table(output_dir=report_dir)
         # Results contains paths to stats and object summaries.
         return results
 
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 7351b913..5be6ead7 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -4,7 +4,7 @@ jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
-pipestat>=0.11.1a1
+pipestat>=0.12.0a1
 peppy>=0.40.6
 pyyaml>=3.12
 rich>=9.10.0
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 85f65f3a..bc23bfb6 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -3,7 +3,11 @@
 import pytest
 from peppy import Project
 
-from looper.exceptions import PipestatConfigurationException, MisconfigurationException, LooperReportError
+from looper.exceptions import (
+    PipestatConfigurationException,
+    MisconfigurationException,
+    LooperReportError,
+)
 from tests.conftest import *
 from looper.cli_pydantic import main
 import pandas as pd
@@ -86,12 +90,12 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
             except Exception:
                 raise pytest.fail("DID RAISE {0}".format(Exception))
         else:
-            with pytest.raises(expected_exception=LooperReportError): # Looper report will and should raise exception if there are no results reported.
+            with pytest.raises(
+                expected_exception=LooperReportError
+            ):  # Looper report will and should raise exception if there are no results reported.
                 result = main(test_args=x)
 
 
-
-
 class TestLooperRerun:
     @pytest.mark.parametrize(
         "flags", [FLAGS[2], FLAGS[3]]

From 47d0f297db6cc8b27208847d688ae2872cc4201d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 30 Dec 2024 13:42:36 -0500
Subject: [PATCH 55/56] update for pre-release 2.0.0a3

---
 docs/changelog.md      | 7 +++++++
 looper/_version.py     | 2 +-
 looper/cli_pydantic.py | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 662481f5..c13d0b07 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,6 +2,13 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [2.0.0a3] -- 2024-12-30
+
+- fix [#536](https://github.com/pepkit/looper/issues/536)
+- fix [#522](https://github.com/pepkit/looper/issues/522)
+- fix [#537](https://github.com/pepkit/looper/issues/537)
+- fix [#534](https://github.com/pepkit/looper/issues/534)
+
 ## [2.0.0a2] -- 2024-09-30
 
 This release breaks backwards compatibility for Looper versions < 2.0.0
diff --git a/looper/_version.py b/looper/_version.py
index 05c48b5c..15d9896c 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1,2 +1,2 @@
-__version__ = "2.0.0a2"
+__version__ = "2.0.0a3"
 # You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 9c66f794..00cf3071 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -340,7 +340,7 @@ def main(test_args=None) -> dict:
         prog="looper",
         description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="2.0.0a2",
+        version="2.0.0a3",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)

From c0141f7eb3f6d95fb26c9b019d01c0aeffc63f9f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 16 Jan 2025 16:27:31 -0500
Subject: [PATCH 56/56] update changelog and versions for 2.0.0 release

---
 docs/changelog.md                 | 13 +++++--------
 looper/_version.py                |  4 ++--
 looper/cli_pydantic.py            |  2 +-
 requirements/requirements-all.txt |  2 +-
 4 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index c76fd636..986d8550 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,14 +2,7 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [2.0.0a3] -- 2024-12-30
-
-- fix [#536](https://github.com/pepkit/looper/issues/536)
-- fix [#522](https://github.com/pepkit/looper/issues/522)
-- fix [#537](https://github.com/pepkit/looper/issues/537)
-- fix [#534](https://github.com/pepkit/looper/issues/534)
-
-## [2.0.0a2] -- 2024-09-30
+## [2.0.0] -- 2025-01-16
 
 This release breaks backwards compatibility for Looper versions < 2.0.0
 
@@ -19,6 +12,10 @@ This release breaks backwards compatibility for Looper versions < 2.0.0
 - looper cli parameters now take priority as originally intended [#518](https://github.com/pepkit/looper/issues/518)
 - fix divvy inspect
 - remove printed dictionary at looper finish [#511](https://github.com/pepkit/looper/issues/511)
+- fix [#536](https://github.com/pepkit/looper/issues/536)
+- fix [#522](https://github.com/pepkit/looper/issues/522)
+- fix [#537](https://github.com/pepkit/looper/issues/537)
+- fix [#534](https://github.com/pepkit/looper/issues/534)
 
 ### Changed
 - `--looper-config` is now `--config`, `-c`. [#455](https://github.com/pepkit/looper/issues/455)
diff --git a/looper/_version.py b/looper/_version.py
index a780b629..0ac2c675 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1,2 +1,2 @@
-__version__ = "2.0.0a3"
-# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
\ No newline at end of file
+__version__ = "2.0.0"
+# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 00cf3071..1bef39f3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -340,7 +340,7 @@ def main(test_args=None) -> dict:
         prog="looper",
         description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="2.0.0a3",
+        version="2.0.0",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 5be6ead7..6533040a 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -1,5 +1,5 @@
 colorama>=0.3.9
-eido>=0.2.3
+eido>=0.2.4
 jinja2
 logmuse>=0.2.0
 pandas>=2.0.2