Merge pull request #42 from NREL/pp/status_upgrades

Status upgrades + Apptainer

Author: ppinchuk

.github/workflows/apptainer-build-deploy.yml

@@ -0,0 +1,55 @@
+name: Apptainer Build and Publish
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-publish-container:
+    name: Build and Publish for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            artifact_name: gaps.sif
+            asset_name: gaps-linux-amd64
+            body: GAPs Apptainer Image (ubuntu-latest)
+        #   - os: windows-latest
+        #     artifact_name: mything.exe
+        #     asset_name: mything-windows-amd64
+        #   - os: macos-latest
+        #     artifact_name: mything
+        #     asset_name: mything-macos-amd64
+    permissions:
+      contents: read
+      packages: write
+
+    container:
+        image: quay.io/singularity/singularity:v3.8.1
+        options: --privileged
+
+    steps:
+
+      - name: Check out code for the container builds
+        uses: actions/checkout@v2
+
+      - name: Build Container
+        run: |
+           singularity build gaps.sif Apptainer
+
+    #   - name: Login and Deploy Container
+    #     run: |
+    #        echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io
+    #        singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag}
+
+      - name: Upload container to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ${{ matrix.artifact_name }}
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}
+          overwrite: true
+          body: ${{ matrix.body }}

Apptainer

@@ -0,0 +1,18 @@
+Bootstrap: docker
+From: python:3.11
+
+%labels
+    Author Paul Pinchuk
+    Maintainer  [email protected]
+    URL https://github.com/NREL/gaps
+
+%post
+    echo "Installing vim"
+    apt-get update && apt-get -y upgrade
+    apt-get -y --allow-unauthenticated install vim
+
+    echo "Installing GAPs..."
+    pip install NREL-gaps
+
+%runscript
+    "$@"

gaps/_cli.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+GAPs CLI entry points.
+"""
+import click
+
+from gaps.version import __version__
+from gaps.cli.status import status_command
+
+
+@click.group()
+@click.version_option(version=__version__)
+@click.pass_context
+def main(ctx):
+    """GAPs command line interface."""
+    ctx.ensure_object(dict)
+
+
+main.add_command(status_command(), name="status")
+
+
+if __name__ == "__main__":
+    # pylint: disable=no-value-for-parameter
+    main(obj={})

gaps/cli/cli.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-Generation CLI entry points.
+Main CLI entry points.
 """
 from functools import partial
 

gaps/cli/execution.py

@@ -4,10 +4,11 @@
 """
 import logging
 import datetime as dt
+from pathlib import Path
 from warnings import warn
 from inspect import signature
 
-from gaps.hpc import submit
+from gaps.hpc import submit, DEFAULT_STDOUT_PATH
 from gaps.status import (
     DT_FMT,
     Status,
@@ -153,6 +154,10 @@ def _kickoff_hpc_job(ctx, cmd, hardware_option, **kwargs):
     id_msg = f" (Job ID #{out})" if out else ""
     msg = f"Kicked off {command!r} job {name!r}{id_msg}"
 
+    stdout_dir = Path(kwargs.get("stdout_path", DEFAULT_STDOUT_PATH))
+    stdout_log_file = str(stdout_dir / f"{name}_{out}.o")
+    stdout_err_log_file = str(stdout_dir / f"{name}_{out}.e")
+
     Status.mark_job_as_submitted(
         ctx.obj["OUT_DIR"],
         pipeline_step=ctx.obj["PIPELINE_STEP"],
@@ -164,6 +169,8 @@ def _kickoff_hpc_job(ctx, cmd, hardware_option, **kwargs):
             StatusField.QOS: kwargs.get("qos") or QOSOption.UNSPECIFIED,
             StatusField.JOB_STATUS: StatusOption.SUBMITTED,
             StatusField.TIME_SUBMITTED: dt.datetime.now().strftime(DT_FMT),
+            StatusField.STDOUT_LOG: stdout_log_file,
+            StatusField.STDOUT_ERR_LOG: stdout_err_log_file,
         },
     )
     logger.info(msg)

gaps/cli/status.py

@@ -321,6 +321,8 @@ def main_monitor(folder, pipe_steps, status, include, recursive):
     for directory in folders:
         if not directory.is_dir():
             continue
+        if directory.name == Status.HIDDEN_SUB_DIR:
+            continue
 
         pipe_status = Status(directory)
         if not pipe_status:

gaps/status.py

@@ -42,6 +42,8 @@ class StatusField(CaseInsensitiveEnum):
     TOTAL_RUNTIME = "total_runtime"
     RUNTIME_SECONDS = "runtime_seconds"
     MONITOR_PID = "monitor_pid"
+    STDOUT_LOG = "stdout_log"
+    STDOUT_ERR_LOG = "stdout_err_log"
 
 
 # pylint: disable=no-member
@@ -865,6 +867,7 @@ def _add_elapsed_time(status_df):
     has_not_failed = status_df[StatusField.JOB_STATUS] != StatusOption.FAILED
     mask = has_start_time & (has_no_end_time & has_not_failed)
 
+    status_df = _add_time_cols_if_needed(status_df)
     start_times = status_df.loc[mask, StatusField.TIME_START]
     start_times = pd.to_datetime(start_times, format=DT_FMT)
     elapsed_times = dt.datetime.now() - start_times
@@ -876,6 +879,14 @@ def _add_elapsed_time(status_df):
     return status_df
 
 
+def _add_time_cols_if_needed(status_df):
+    """Adds any missing time cols to avoid pandas 2.0 warnings"""
+    for col in [StatusField.RUNTIME_SECONDS, StatusField.TOTAL_RUNTIME]:
+        if col not in status_df:
+            status_df[col] = None
+    return status_df
+
+
 def _load(fpath):
     """Load status json."""
     if fpath.is_file():

gaps/version.py

@@ -1,3 +1,3 @@
 """GAPs Version Number. """
 
-__version__ = "0.6.6"
+__version__ = "0.6.7"

setup.py

@@ -59,4 +59,5 @@
         "dev": TEST_REQUIREMENTS + DEV_REQUIREMENTS,
         "docs": TEST_REQUIREMENTS + DEV_REQUIREMENTS + DOC_REQUIREMENTS,
     },
+    entry_points={"console_scripts": ["gaps=gaps._cli:main"]},
 )

tests/cli/test_cli_execution.py

@@ -187,6 +187,8 @@ def _test_submit(cmd):
         status = json.load(status_fh)
 
     assert status["run"][job_name][StatusField.HARDWARE] == "eagle"
+    assert "9999.o" in status["run"][job_name][StatusField.STDOUT_LOG]
+    assert "9999.e" in status["run"][job_name][StatusField.STDOUT_ERR_LOG]
     if high_qos:
         assert status["run"][job_name][StatusField.QOS] == "high"
     else:

tests/cli/test_cli_status.py

@@ -5,12 +5,14 @@
 import json
 import shutil
 from pathlib import Path
+from contextlib import nullcontext
 
 import psutil
 import pytest
 
 from gaps.status import HardwareStatusRetriever, StatusOption, Status
 from gaps.cli.status import status_command
+from gaps._cli import main
 from gaps.warnings import gapsWarning
 
 
@@ -36,7 +38,10 @@
         + "-s dne".split(),
     ],
 )
-def test_status(test_data_dir, cli_runner, extra_args, monkeypatch):
+@pytest.mark.parametrize("test_main_entry", [True, False])
+def test_status(
+    test_data_dir, cli_runner, extra_args, test_main_entry, monkeypatch
+):
     """Test the status command."""
 
     monkeypatch.setattr(psutil, "pid_exists", lambda *__: True, raising=True)
@@ -47,18 +52,23 @@ def test_status(test_data_dir, cli_runner, extra_args, monkeypatch):
         raising=True,
     )
 
-    status = status_command()
+    if test_main_entry:
+        status = main
+        command_args = ["status"]
+    else:
+        status = status_command()
+        command_args = []
+
+    command_args += [(test_data_dir / "test_run").as_posix()] + extra_args
+
     if "dne" in extra_args:
-        with pytest.warns(gapsWarning):
-            result = cli_runner.invoke(
-                status,
-                [(test_data_dir / "test_run").as_posix()] + extra_args,
-            )
+        expected_behavior = pytest.warns(gapsWarning)
     else:
-        result = cli_runner.invoke(
-            status,
-            [(test_data_dir / "test_run").as_posix()] + extra_args,
-        )
+        expected_behavior = nullcontext()
+
+    with expected_behavior:
+        result = cli_runner.invoke(status, command_args)
+
     lines = result.stdout.split("\n")
     cols = [
         "job_status",
@@ -299,6 +309,7 @@ def test_recursive_status(tmp_path, test_data_dir, cli_runner, monkeypatch):
     assert any(line == "test_run:" for line in lines)
     assert any(line == "test_failed_run:" for line in lines)
     assert len(lines) > 20
+    assert not any(Status.HIDDEN_SUB_DIR in line for line in lines)
 
 
 if __name__ == "__main__":