Automate the verify tools to submit from start pdy to end pdy

utils/soca/fig_gallery/README.md

@@ -0,0 +1,38 @@
+## How to generate the EVA and State space figures
+
+#### Create a scratch place to run `run_vrfy.py`. This script will generate a bunch of sbatch scripts and logs.
+```
+mkdir /somewhere/scratch
+cd /somewhere/scratch
+ln -s /path/to/run_vrfy.py .   # to be sorted out properly in the future
+cp /path/to/vrfy_config.yaml .
+module use ...
+module load EVA/....
+```
+---
+#### Edit `vrfy_config.yaml`
+It's actually read as a jinja template to render `pslot` if necessary. Anything that is a templated variable in `vrfy_jobcard.sh.j2` can be added to the yaml below.
+```yaml
+pslot: "nomlb"
+start_pdy: '20210701'
+end_pdy: '20210701'
+cycs: ["00", "06", "12", "18"]
+run: "gdas"
+homegdas: "/work2/noaa/da/gvernier/runs/mlb/GDASApp"
+base_exp_path: "/work2/noaa/da/gvernier/runs/mlb/{{ pslot }}/COMROOT/{{ pslot }}"
+plot_ensemble_b: "OFF"
+plot_parametric_b: "OFF"
+plot_background: "OFF"
+plot_increment: "ON"
+plot_analysis: "OFF"
+eva_plots: "ON"
+qos: "batch"
+hpc: "hercules"
+eva_module: "EVA/orion"
+```
+
+---
+#### Run the application
+```python run_vrfy.py vrfy_config.yaml```
+This will generate and submit the job cards for all the **cycles** defined by `cycs`, from `start_pdy` to `end_pdy`.
+

utils/soca/fig_gallery/run_vrfy.py

@@ -0,0 +1,71 @@
+from jinja2 import Template
+import subprocess
+from datetime import datetime, timedelta
+import yaml
+import sys
+import copy
+import os
+
+def iterate_pdy_range(start_pdy, end_pdy):
+    """Generate a range of dates in YYYYMMDD format."""
+    start_date = datetime.strptime(start_pdy, "%Y%m%d")
+    end_date = datetime.strptime(end_pdy, "%Y%m%d")
+    current_date = start_date
+
+    while current_date <= end_date:
+        yield current_date.strftime("%Y%m%d")
+        current_date += timedelta(days=1)
+
+
+def generate_jobcard(template_path, output_path, context):
+    # Read the Jinja2 template file
+    with open(template_path, 'r') as file:
+        template_content = file.read()
+
+    # Create a Jinja2 template object
+    template = Template(template_content)
+
+    # Render the template with custom values
+    rendered_script = template.render(**context)
+
+    # Write the rendered script to the output file
+    with open(output_path, 'w') as file:
+        file.write(rendered_script)
+
+    print(f"Bash script generated at: {output_path}")
+
+# Example usage
+if __name__ == "__main__":
+
+    # Get the YAML configuration file name from the input argument
+    if len(sys.argv) != 2:
+        print("Usage: python run_vrfy.py <config.yaml>")
+        sys.exit(1)
+
+    config_file = sys.argv[1]
+
+    # Read the YAML template from the file
+    with open(config_file, "r") as file:
+        yaml_template = file.read()
+
+    # Load the template YAML as a dictionary
+    template_dict = yaml.safe_load(yaml_template)
+
+    # Render the template with Jinja2
+    template = Template(yaml_template)
+    config = yaml.safe_load(template.render(pslot=template_dict["pslot"]))
+
+    # Iterate over the date range
+    for pdy in iterate_pdy_range(config['start_pdy'], config['end_pdy']):
+        context = copy.deepcopy(config)
+        for cyc in config["cycs"]:
+          # Update the cycle's date
+          context.update({"pdy": pdy, "cyc": cyc})
+
+          # Prepare the job card
+          template_jobcard = os.path.join(context['homegdas'], 'utils', 'soca', 'fig_gallery', 'vrfy_jobcard.sh.j2')  # Assumes a Jinja2 template file in the moegdas directory
+          jobcard = f"vrfy_jobcard.{context['pslot']}.{context['pdy']}.{context['cyc']}.sh"
+          generate_jobcard(template_jobcard, jobcard, context)
+
+          # Submit the plotting job
+          subprocess.run(f"sbatch {jobcard}", shell=True)

utils/soca/fig_gallery/vrfy_config.yaml

@@ -0,0 +1,16 @@
+pslot: "nomlb"
+start_pdy: '20210701'
+end_pdy: '20210701'
+cycs: ["00", "06", "12", "18"]
+run: "gdas"
+homegdas: "/work2/noaa/da/gvernier/runs/mlb/GDASApp"
+base_exp_path: "/work2/noaa/da/gvernier/runs/mlb/{{ pslot }}/COMROOT/{{ pslot }}"
+plot_ensemble_b: "OFF"
+plot_parametric_b: "OFF"
+plot_background: "OFF"
+plot_increment: "ON"
+plot_analysis: "OFF"
+eva_plots: "ON"
+qos: "batch"
+hpc: "hercules"
+eva_module: "EVA/orion"

utils/soca/fig_gallery/vrfy_jobcard.sh.j2

@@ -0,0 +1,58 @@
+#!/bin/bash
+#SBATCH --job-name={{ job_name | default("marine_vrfy") }}    # Assign a name to the job (customize as needed)
+#SBATCH --account={{ account | default("da-cpu") }}
+#SBATCH --qos={{ qos | default("debug") }}
+{% set OUTPUT = "vrfy_jobcard." + pslot + "." + pdy + "." + cyc + ".log" %}
+#SBATCH --output={{ OUTPUT }}
+#SBATCH --nodes={{ nodes | default(1) }}                  # Request 1 node
+#SBATCH --tasks={{ ntasks | default(20) }}               # Request total tasks (processors across nodes)
+{% set HPC = hpc | default("hera") %}
+{% if HPC == "hera" %}
+#SBATCH --partition={{ partition | default("hera") }}     # Specify the partition (cluster)
+{% endif %}
+#SBATCH --mem={{ memory | default("24GB") }}              # Request memory
+#SBATCH --time={{ walltime | default("00:30:00") }}       # Set the walltime limit
+
+# Define HOMEgdas
+export HOMEgdas="{{ homegdas }}"
+
+# Load EVA module
+module use ${HOMEgdas}/modulefiles
+module load {{ eva_module | default("EVA/hera") }}
+
+# Set PYTHONPATH using HOMEgfs
+export PYTHONPATH="${HOMEgdas}/ush/:\
+${HOMEgdas}/ush/eva/:\
+${HOMEgdas}/ush/soca/:\
+$PYTHONPATH"
+
+# Set flags to control plotConfig in the Python script
+export PLOT_ENSEMBLE_B={{ plot_ensemble_b | default("OFF") }}
+export PLOT_PARAMETRIC_B={{ plot_parametric_b | default("ON") }}
+export PLOT_BACKGROUND={{ plot_background | default("ON") }}
+export PLOT_INCREMENT={{ plot_increment | default("ON") }}
+export PLOT_ANALYSIS={{ plot_analysis | default("OFF") }}
+export PLOT_ANALYSIS={{ plot_analysis | default("OFF") }}
+export EVA_PLOTS={{ eva_plots | default("OFF") }}
+
+# Define and export the environment variables
+export cyc="{{ cyc }}"
+export RUN="{{ run | default("gdas") }}"
+export PSLOT="{{ pslot }}"
+export PDY="{{ pdy }}"
+
+# Define base experiment path
+BASE_EXP_PATH="{{ base_exp_path }}"  # path to the gdas.pdy directory
+
+# Calculate previous date and cycle
+PREV_CYC=$(date -d "{{ pdy }} {{ cyc }} -6 hours" +"%Y%m%d %H")
+PREV_PDY=$(echo $PREV_CYC | cut -d' ' -f1)
+PREV_CYC_HOUR=$(echo $PREV_CYC | cut -d' ' -f2)
+
+# Define and export environment variables with paths
+export COM_OCEAN_ANALYSIS="${BASE_EXP_PATH}/gdas.{{ pdy }}/{{ cyc }}/analysis/ocean"
+export COM_ICE_HISTORY_PREV="${BASE_EXP_PATH}/gdas.${PREV_PDY}/${PREV_CYC_HOUR}/model/ice/history"
+export COM_OCEAN_HISTORY_PREV="${BASE_EXP_PATH}/gdas.${PREV_PDY}/${PREV_CYC_HOUR}/model/ocean/history"
+
+# Execute Marine Verify Analysis
+python3 ${HOMEgdas}/utils/soca/fig_gallery/vrfy_script.py