From 9306bb5823d83e12c3f80e3ebe15b2c0fa3bbc1a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 11 Aug 2024 22:36:07 -0400 Subject: [PATCH 01/67] remove error message from archive.py --- sorc/link_workflow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 9722f5a2b8e..be912292fee 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -71,8 +71,8 @@ ${LINK_OR_COPY} "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/ru case "${machine}" in "wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;; "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; - "orion") FIX_DIR="/work/noaa/global/kfriedma/glopara/fix" ;; - "hercules") FIX_DIR="/work/noaa/global/kfriedma/glopara/fix" ;; + "orion") FIX_DIR="/work/noaa/global/glopara/fix" ;; + "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; From 9e43bda87062cc4e83081a06ea4af32c6a8588be Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 13 Aug 2024 16:33:22 -0400 Subject: [PATCH 02/67] test1 --- parm/archive/gefs_arcdir.yaml.j2 | 22 ++++++++++++++++++++++ ush/python/pygfs/task/archive.py | 5 +---- 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 parm/archive/gefs_arcdir.yaml.j2 diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 new file mode 100644 index 00000000000..e2f4dcebb89 --- /dev/null +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -0,0 +1,22 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set head = RUN + ".t" + cycle_HH + "z." %} + +# Declare the GEFS_ARCH where atmos data will be sent +{% set GEFS_ARCH = ROTDIR ~ RUN ~"/arch" %} + +# test files +{% if RUN == "gefs" %} + {% set det_files = [] %} + {% if path_exists(ROTDIR ~ RUN ~ "." ~ cycle_YMD ~ cycle_HH + ~ "/ensstat/products/atmos/grib2/0p50") %} + {% do det_files.append([ROTDIR ~ RUN ~ "." ~ cycle_YMD ~ cycle_HH + ~ "/ensstat/products/atmos/grib2/0p50/gefs.t12z.mean.pres_.0p50.f006.grib2"]) %} + {% endif %} + +# Actually write the yaml +mkdir: + - "{{ GEFS_ARCH }}" +copy: + - {{ det_files }} diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 953a856192e..25651ba7787 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -63,9 +63,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - if arch_dict.RUN == "gefs": - raise NotImplementedError("FATAL ERROR: Archiving is not yet set up for GEFS runs") - if arch_dict.RUN in ["gdas", "gfs"]: # Copy the cyclone track files and rename the experiments @@ -75,7 +72,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "arcdir.yaml.j2") + arcdir_j2yaml = os.path.join(archive_parm, "gefs_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit From 2f6b9e11c18dedc4849828c470a3018be65bd807 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 14 Aug 2024 11:36:15 -0400 Subject: [PATCH 03/67] updated gefes_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f0f73d11737..3d41686faef 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -617,7 +617,7 @@ def arch(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' From f94fc052058077d991ecae50c275a43b1edf4c43 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 16 Aug 2024 17:07:20 -0400 Subject: [PATCH 04/67] update gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index e2f4dcebb89..454cc67ea47 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -4,19 +4,22 @@ {% set head = RUN + ".t" + cycle_HH + "z." %} # Declare the GEFS_ARCH where atmos data will be sent -{% set GEFS_ARCH = ROTDIR ~ RUN ~"/arch" %} +{% set GEFS_ARCH = ROTDIR ~"/gefsarch" %} # test files {% if RUN == "gefs" %} {% set det_files = [] %} - {% if path_exists(ROTDIR ~ RUN ~ "." ~ cycle_YMD ~ cycle_HH - ~ "/ensstat/products/atmos/grib2/0p50") %} - {% do det_files.append([ROTDIR ~ RUN ~ "." ~ cycle_YMD ~ cycle_HH - ~ "/ensstat/products/atmos/grib2/0p50/gefs.t12z.mean.pres_.0p50.f006.grib2"]) %} + {% if path_exists(ROTDIR ~ "/gefs.20210323/" ~ "12" + ~ "/ensstat/products/atmos/grib2/0p50") %} + {% do det_files.append([ROTDIR ~ "/gefs.20210323/" ~ "12" + ~ "/ensstat/products/atmos/grib2/0p50/gefs.t12z.mean.pres_.0p50.f000.grib2.idx", GEFS_ARCH]) %} {% endif %} - +{% endif %} +{% set file_set = det_files %} # Actually write the yaml mkdir: - "{{ GEFS_ARCH }}" copy: - - {{ det_files }} + {% for source_dest_pair in file_set %} + - {{ source_dest_pair }} + {% endfor %} From ef4d37c49979b6415b99984c1b3ed0b167f60563 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 16 Aug 2024 17:20:52 -0400 Subject: [PATCH 05/67] updated config.arch --- parm/config/gefs/config.arch | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/parm/config/gefs/config.arch b/parm/config/gefs/config.arch index a23bcce6aeb..5f482605155 100644 --- a/parm/config/gefs/config.arch +++ b/parm/config/gefs/config.arch @@ -11,5 +11,25 @@ echo "BEGIN: config.arch" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} +#GEFS +export DO_VERFRAD="NO" +export DO_VMINMON="NO" # GSI minimization monitoring +export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 +export DO_VERFOZN="NO" +export FHMAX_WAV_GFS=240 +export restart_interval_gdas=6 +export restart_interval_gfs=6 +export AERO_ANL_RUN="NO" +export AERO_FCST_RUN="NO" +export DOIBP_WAV="NO" +export DO_JEDIOCNVAR="NO" +export NMEM_ENS=20 +export DO_JEDIATMVAR="NO" +export DO_VRFY_OCEANDA="NO" +export FHMAX_FITS=240 +export IAUFHRS=6 +export DO_FIT2OBS="NO" +export DO_PREP_OBS_AERO="NO" +export DOHYBVAR="NO" echo "END: config.arch" From dda46b78f53119b72c202e51496546114d4e5a7a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 21 Aug 2024 14:41:36 -0400 Subject: [PATCH 06/67] update archive.py --- parm/archive/gefs_arcdir.yaml.j2 | 13 +++++++------ ush/python/pygfs/task/archive.py | 5 +++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 454cc67ea47..f6800f1def7 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -6,16 +6,17 @@ # Declare the GEFS_ARCH where atmos data will be sent {% set GEFS_ARCH = ROTDIR ~"/gefsarch" %} +{% set file_set = [] %} + # test files {% if RUN == "gefs" %} - {% set det_files = [] %} - {% if path_exists(ROTDIR ~ "/gefs.20210323/" ~ "12" - ~ "/ensstat/products/atmos/grib2/0p50") %} - {% do det_files.append([ROTDIR ~ "/gefs.20210323/" ~ "12" - ~ "/ensstat/products/atmos/grib2/0p50/gefs.t12z.mean.pres_.0p50.f000.grib2.idx", GEFS_ARCH]) %} + {% set atmos_files = [] %} + {% if path_exists(ROTDIR ~ "/gefs." ~ cycle_YMD ~ "12/ensstat/products/atmos/grib2/1p00") %} + {% do atmos_files.append([ROTDIR ~ "/gefs." ~ cycle_YMD ~ + ~ "12/ensstat/products/atmos/grib2/1p00/gefs.t12z.mean.pres_.0p50.f000.grib2.idx", GEFS_ARCH]) %} {% endif %} {% endif %} -{% set file_set = det_files %} +{% set file_set = atmos_files %} # Actually write the yaml mkdir: - "{{ GEFS_ARCH }}" diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 25651ba7787..55801c8208a 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -71,8 +71,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") - # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "gefs_arcdir.yaml.j2") + if arch_dict.RUN == "gefs": + # Collect the dataset to archive locally + arcdir_j2yaml = os.path.join(archive_parm, "gefs_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit From 58fdd37b3c1421ff2f703e273d57ae71fba2e09f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 13:07:48 -0400 Subject: [PATCH 07/67] updated JGLOBAL_ARCHIVE and gefs_arch.yaml.j2 --- jobs/JGLOBAL_ARCHIVE | 1 + parm/archive/gefs_arcdir.yaml.j2 | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 401feba35f6..56653918d5f 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -14,6 +14,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ + COMIN_TOP:COM_TOP_TMPL \ COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index f6800f1def7..a5bfc5a2779 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -10,13 +10,16 @@ # test files {% if RUN == "gefs" %} - {% set atmos_files = [] %} - {% if path_exists(ROTDIR ~ "/gefs." ~ cycle_YMD ~ "12/ensstat/products/atmos/grib2/1p00") %} - {% do atmos_files.append([ROTDIR ~ "/gefs." ~ cycle_YMD ~ - ~ "12/ensstat/products/atmos/grib2/1p00/gefs.t12z.mean.pres_.0p50.f000.grib2.idx", GEFS_ARCH]) %} + {% set ensstat_files = [] %} + {% if path_exists(COMIN_ATMOS_GRIB_1p00) %} + {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} + {% do ensstat_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "/" ~ + grid ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", + GEFS_ARCH]) %} + {% endfor %} {% endif %} {% endif %} -{% set file_set = atmos_files %} +{% set file_set = ensstat_files %} # Actually write the yaml mkdir: - "{{ GEFS_ARCH }}" From c50cf8b706ef86a21eecea81142abb8130ac6cdf Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 13:23:50 -0400 Subject: [PATCH 08/67] updated config.com --- jobs/JGLOBAL_ARCHIVE | 4 ++++ parm/config/gfs/config.com | 1 + 2 files changed, 5 insertions(+) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 56653918d5f..a696faca3b5 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -51,6 +51,10 @@ for grid in "0p25" "0p50" "1p00"; do "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done +for grid in "0p25" "0p50" "1p00"; do + YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ + "COMIN_ATMOS_ENSSTAT_${grid}:COM_TOP_TMPL" +done ############################################################### # Run archive script ############################################################### diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 02a5b1edf57..0bd1385a06e 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -69,6 +69,7 @@ declare -rx COM_ATMOS_OZNMON_TMPL=${COM_BASE}'/products/atmos/oznmon' declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' +declare -rx COM_ATMOS_ENSSTAT_TMPL=${COM_TOP_TMPL}'/ensstat/products/atmos/grib2' declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model/wave/prep' From 652e779bb66f1008c56c0cd9f6165644033a91d7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 13:44:44 -0400 Subject: [PATCH 09/67] updated JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index a696faca3b5..1ede44e21c3 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -13,8 +13,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMIN_ATMOS_ENSSTAT:COM_ATMOS_ENSSTAT_TMPL \ COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ - COMIN_TOP:COM_TOP_TMPL \ COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ @@ -53,8 +53,9 @@ done for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ - "COMIN_ATMOS_ENSSTAT_${grid}:COM_TOP_TMPL" + "COMIN_ATMOS_ENSSTAT_${grid}:COM_ATMOS_ENSSTAT_TMPL" done + ############################################################### # Run archive script ############################################################### From 99df46eeecd44e2cd0ec4b03ce84c01a132fc84f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 13:45:58 -0400 Subject: [PATCH 10/67] updated gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index a5bfc5a2779..069346779bf 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -11,9 +11,9 @@ # test files {% if RUN == "gefs" %} {% set ensstat_files = [] %} - {% if path_exists(COMIN_ATMOS_GRIB_1p00) %} + {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "/" ~ + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "/" ~ grid ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From f4bbcea2331a9fdc7e22b82582e5e29a9e95689b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 14:09:45 -0400 Subject: [PATCH 11/67] updated gefs_arch.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 069346779bf..4d85b022192 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -14,7 +14,7 @@ {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "/" ~ - grid ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", + "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} {% endif %} From 13c4e5616a1fbb79804d46aabaf8286e8c910b59 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 14:17:16 -0400 Subject: [PATCH 12/67] modified: parm/archive/gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 4d85b022192..3946c6118d1 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -13,7 +13,7 @@ {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "/" ~ + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From 7746024c11f0a010305356917a3b13b429c89710 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 14:21:55 -0400 Subject: [PATCH 13/67] updated gefs_Arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 3946c6118d1..73eb0b68558 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -13,7 +13,7 @@ {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/1p00/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From fdd4937faea05c5bca99cfce6284111b90eeb9f0 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 14:59:24 -0400 Subject: [PATCH 14/67] updated gefs_Arch.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 73eb0b68558..e01fffcebe4 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -12,7 +12,7 @@ {% if RUN == "gefs" %} {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} - {% for fhr in range(FHMIN_GFS, FHMIN_GFS + FHOUT_GFS, FHMAX_GFS) %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS, FHOUT_GFS + FHMAX_GFS) %} {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/1p00/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} From 484cfb5cd9d87138e4b363ec5a383a324004ad03 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 22 Aug 2024 20:57:04 -0400 Subject: [PATCH 15/67] updated gefs_arc.yamlj2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index e01fffcebe4..1faa5cc7b01 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -12,7 +12,7 @@ {% if RUN == "gefs" %} {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS, FHOUT_GFS + FHMAX_GFS) %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS+1, FHOUT_GFS) %} {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/1p00/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} From bb0cc1df276345d5cefd459d1f540d371dc009e6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 14:08:07 -0400 Subject: [PATCH 16/67] modified archive.py --- ush/python/pygfs/task/archive.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 55801c8208a..ddcd24ca9c9 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -71,9 +71,8 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") - if arch_dict.RUN == "gefs": - # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "gefs_arcdir.yaml.j2") + # Collect the dataset to archive locally + arcdir_j2yaml = os.path.join(archive_parm, "{arch_dict.RUN}_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit From bdc5851a32a9a43e8ae20197480bf7e50360895a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 14:33:09 -0400 Subject: [PATCH 17/67] modify exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index ec8154317f3..49c09bc2bcd 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -35,7 +35,7 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config[key, "undefined"] # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): From 590611225b133bc1873c8a2012ebaf58d531cde7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 14:37:36 -0400 Subject: [PATCH 18/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 49c09bc2bcd..fadee7cd1a1 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -35,7 +35,7 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key, "undefined"] + archive_dict[key] = archive.task_config[key, "Undefined"] # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): From 2dbcc80ab50056b3996daa1fe21c01b28d4a034e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 14:38:48 -0400 Subject: [PATCH 19/67] Remove config.arch --- parm/config/gefs/config.arch | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/parm/config/gefs/config.arch b/parm/config/gefs/config.arch index 5f482605155..a23bcce6aeb 100644 --- a/parm/config/gefs/config.arch +++ b/parm/config/gefs/config.arch @@ -11,25 +11,5 @@ echo "BEGIN: config.arch" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} -#GEFS -export DO_VERFRAD="NO" -export DO_VMINMON="NO" # GSI minimization monitoring -export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 -export DO_VERFOZN="NO" -export FHMAX_WAV_GFS=240 -export restart_interval_gdas=6 -export restart_interval_gfs=6 -export AERO_ANL_RUN="NO" -export AERO_FCST_RUN="NO" -export DOIBP_WAV="NO" -export DO_JEDIOCNVAR="NO" -export NMEM_ENS=20 -export DO_JEDIATMVAR="NO" -export DO_VRFY_OCEANDA="NO" -export FHMAX_FITS=240 -export IAUFHRS=6 -export DO_FIT2OBS="NO" -export DO_PREP_OBS_AERO="NO" -export DOHYBVAR="NO" echo "END: config.arch" From 4bfcd6216c73872ebddd6d54898d28f6ce72971f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 14:45:34 -0400 Subject: [PATCH 20/67] updated archive.py --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index ddcd24ca9c9..dfff8b2c776 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -72,7 +72,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "{arch_dict.RUN}_arcdir.yaml.j2") + arcdir_j2yaml = os.path.join(archive_parm, f"{arch_dict.RUN}_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit From deb38d29df9aed80d50f4e7670d1a48269048a6b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 15:38:59 -0400 Subject: [PATCH 21/67] modified JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 1ede44e21c3..9d216b94b74 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -51,11 +51,6 @@ for grid in "0p25" "0p50" "1p00"; do "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done -for grid in "0p25" "0p50" "1p00"; do - YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ - "COMIN_ATMOS_ENSSTAT_${grid}:COM_ATMOS_ENSSTAT_TMPL" -done - ############################################################### # Run archive script ############################################################### From 68d8452308e36cdd56b56894a10e9a118e064951 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 15:40:57 -0400 Subject: [PATCH 22/67] modified gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 1faa5cc7b01..d805382d7e1 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -3,6 +3,14 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set head = RUN + ".t" + cycle_HH + "z." %} +{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${MEMDIR}': "ensstat" }) %} + +{% set COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 | replace_tmpl(tmpl_dict) %} + # Declare the GEFS_ARCH where atmos data will be sent {% set GEFS_ARCH = ROTDIR ~"/gefsarch" %} From a293160ebb5d28e61f46642177b4fbd445a36358 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 15:42:30 -0400 Subject: [PATCH 23/67] modified JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 9d216b94b74..401feba35f6 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -13,7 +13,6 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ - COMIN_ATMOS_ENSSTAT:COM_ATMOS_ENSSTAT_TMPL \ COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ From 773d4ba3de1eb06988dbb8780b64833056ee2a29 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 15:44:10 -0400 Subject: [PATCH 24/67] modified config.com parm/gfs --- parm/config/gfs/config.com | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 0bd1385a06e..02a5b1edf57 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -69,7 +69,6 @@ declare -rx COM_ATMOS_OZNMON_TMPL=${COM_BASE}'/products/atmos/oznmon' declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' -declare -rx COM_ATMOS_ENSSTAT_TMPL=${COM_TOP_TMPL}'/ensstat/products/atmos/grib2' declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model/wave/prep' From 7d6822666c894295a5a2a2975f400b7db9afad25 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 15:46:51 -0400 Subject: [PATCH 25/67] renamed archdir.yaml.j2 --- parm/archive/gfs_arcdir.yaml.j2 | 156 ++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 parm/archive/gfs_arcdir.yaml.j2 diff --git a/parm/archive/gfs_arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 new file mode 100644 index 00000000000..57dbc78885e --- /dev/null +++ b/parm/archive/gfs_arcdir.yaml.j2 @@ -0,0 +1,156 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set head = RUN + ".t" + cycle_HH + "z." %} + +# Select data to store in the ARCDIR and VFYARC from deterministic runs +# This file set will contain all source-destination pairs to send to the FileHandler for copying +{% set file_set = [] %} + +# Declare the VFYARC where Fit2Obs data will be sent +{% set VFYARC = ROTDIR ~ "/vrfyarch" %} + +# Deterministic files +{% if "enkf" not in RUN %} + # Common files to be added to both the gfs and gdas keys below + {% set det_files = [] %} + # Cyclone forecasts, produced for both gdas and gfs cycles + ## Only created if tracking is on and there were systems to track + {% if path_exists(COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH, + ARCDIR ~"/atcfunix." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH, + ARCDIR ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH]) %} + {% endif %} + + # Cyclone tracking data + {% for basin in ["epac", "natl"] %} + {% if path_exists(COMIN_ATMOS_TRACK + "/" + basin) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/" ~ basin, + ARCDIR ~ "/" ~ basin ]) %} + {% endif %} + {% endfor %} + + # Deterministic analysis files (generated for cycled experiments) + {% set det_anl_files = [] %} + # Analysis data (if we are running in cycled mode) + {% do det_anl_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.anl", + ARCDIR ~ "/pgbanl." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + + {% if DO_JEDIATMVAR == True %} + {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "atmstat", + ARCDIR ~ "/atmstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% else %} + {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "gsistat", + ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% endif %} + + {% if DO_JEDISNOWDA == True %} + {% do det_anl_files.append([COMIN_SNOW_ANALYSIS ~ "/" ~ head ~ "snowstat.tgz", + ARCDIR ~ "/snowstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".tgz"]) %} + {% endif %} + + {% if AERO_ANL_RUN == RUN or AERO_ANL_RUN == "both" %} + {% do det_anl_files.append([COMIN_CHEM_ANALYSIS ~ "/" ~ head ~ "aerostat", + ARCDIR ~ "/aerostat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% endif %} + + {% if DO_PREP_OBS_AERO == True %} + {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroobs", + ARCDIR ~ "/aeroobs." ~ RUN ~ "." ~ cycle_YMDH]) %} + {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroawobs", + ARCDIR ~ "/aeroawobs." ~ RUN ~ "." ~ cycle_YMDH]) %} + {% endif %} + + # GFS-specific files + {% set gfs_files = [] %} + {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} + {% do gfs_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), + ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + {% endfor %} + + # Cyclone genesis data (only present if there are storms) + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH, + ARCDIR ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH ]) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH, + ARCDIR ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH ]) %} + {% endif %} + + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH, + ARCDIR ~ "/trak.gfso.atcfunix." ~ cycle_YMDH ]) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH, + ARCDIR ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH ]) %} + {% endif %} + + # GFS Fit2Obs data + {% set fit2obs_files = [] %} + {% for fhr in range(0, FHMAX_FITS + 1, 6) %} + {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sfcfile, + VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sfcfile ]) %} + {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sigfile, + VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sigfile ]) %} + {% endfor %} + + # GDAS-specific files + {% set gdas_files = [] %} + {% for fhr in range(0, FHMAX + 1, FHOUT) %} + {% do gdas_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), + ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + {% endfor %} + + # Now append the necessary file pairs to file_set + # Common deterministic files + {% set file_set = file_set + det_files %} + {% if MODE == "cycled" %} + {% set file_set = file_set + det_anl_files %} + {% endif %} + + # Run-specific deterministic files + {% if RUN == "gfs" %} + {% set file_set = file_set + gfs_files %} + # Fit2Obs files + {% if DO_FIT2OBS == True %} + {% set file_set = file_set + fit2obs_files %} + {% endif %} + {% elif RUN == "gdas" %} + {% set file_set = file_set + gdas_files %} + {% endif %} + +{% else %} # End of deterministic files + + # Ensemble analysis files + {% set enkf_files = [] %} + {% if DO_JEDIATMENS == True %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atmensstat", + ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atminc.ensmean.nc", + ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean.nc"]) %} + {% else %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat", + ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean", + ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} + {% endif %} + + # Construct the final file set + {% set file_set = file_set + enkf_files %} + +{% endif %} + + +# Actually write the yaml +mkdir: + - "{{ ARCDIR }}" + + {% if DO_FIT2OBS == True %} + - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" + {% endif %} + +copy: + {% for source_dest_pair in file_set %} + - {{ source_dest_pair }} + {% endfor %} From 0144216fe0b7f962243df9526dd91a3b6f9143a1 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 27 Aug 2024 16:10:32 -0400 Subject: [PATCH 26/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index fadee7cd1a1..13f0ea3baa9 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -40,7 +40,7 @@ def main(): # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get[key] cwd = os.getcwd() From d7f89ca4b2739756f752059074a48442a8314b31 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 13:18:08 -0400 Subject: [PATCH 27/67] updated gefs_archdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index d805382d7e1..2068fae5b24 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -7,12 +7,12 @@ '${RUN}':RUN, '${YMD}':cycle_YMD, '${HH}':cycle_HH, - '${MEMDIR}': "ensstat" }) %} + '${MEMDIR}': 'ensstat' }) %} {% set COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 | replace_tmpl(tmpl_dict) %} - +{% echo COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 %} # Declare the GEFS_ARCH where atmos data will be sent -{% set GEFS_ARCH = ROTDIR ~"/gefsarch" %} +{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %} {% set file_set = [] %} @@ -21,7 +21,7 @@ {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMAX_GFS+1, FHOUT_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/1p00/" ~ head ~ "mean.pres_." ~ + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From c117574fe168f6fa51cec995ee3179e42ef15c6a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 13:19:02 -0400 Subject: [PATCH 28/67] updated gefs_archdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 2068fae5b24..c504c171c7c 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -10,7 +10,7 @@ '${MEMDIR}': 'ensstat' }) %} {% set COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 | replace_tmpl(tmpl_dict) %} -{% echo COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 %} +{% echo COMIN_ATMOS_ENSSTAT_1p00 %} # Declare the GEFS_ARCH where atmos data will be sent {% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %} From ed1913ef6fa35da43ccd2e790b035a6f365fd31b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:01:12 -0400 Subject: [PATCH 29/67] modified gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index c504c171c7c..410852bc4ba 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -3,14 +3,6 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set head = RUN + ".t" + cycle_HH + "z." %} -{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':cycle_YMD, - '${HH}':cycle_HH, - '${MEMDIR}': 'ensstat' }) %} - -{% set COMIN_ATMOS_ENSSTAT_1p00 = COMIN_ATMOS_GRIB_1p00 | replace_tmpl(tmpl_dict) %} -{% echo COMIN_ATMOS_ENSSTAT_1p00 %} # Declare the GEFS_ARCH where atmos data will be sent {% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %} @@ -19,9 +11,9 @@ # test files {% if RUN == "gefs" %} {% set ensstat_files = [] %} - {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS+1, FHOUT_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ head ~ "mean.pres_." ~ + {% if path_exists(COMOUT_ATMOS_ENSSTAT_1p00) %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + {% do ensstat_files.append([COMOUT_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From 39b56f54ba5f8ddb629a88bd1d9a93853f6be5c6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:05:06 -0400 Subject: [PATCH 30/67] modified JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 401feba35f6..b6e8b952cd4 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -50,6 +50,10 @@ for grid in "0p25" "0p50" "1p00"; do "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done +for grid in '0p25' '0p50' '1p00'; do + prod_dir="COMOUT_ATMOS_ENSSTAT_${grid}" + MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" +done ############################################################### # Run archive script ############################################################### From 49f39a210b2f8537fc1cb887e4028679c6cd11b6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:12:36 -0400 Subject: [PATCH 31/67] updated exglobal_archive.py --- scripts/exglobal_archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 13f0ea3baa9..ea8d195da8e 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -35,12 +35,12 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key, "Undefined"] + archive_dict[key] = archive.task_config.get(key, "Undefined") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config.get[key] + archive_dict[key] = archive.task_config[key] cwd = os.getcwd() From 4cfdaebd5af99c3af173ab395acd46686b948ba6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:46:51 -0400 Subject: [PATCH 32/67] modified exglobal_Archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index ea8d195da8e..eed87658605 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -40,7 +40,7 @@ def main(): # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get[key, "Undefined"] cwd = os.getcwd() From 608ebd7394921d187b7d30631862a27d53f3ee31 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:49:44 -0400 Subject: [PATCH 33/67] updated archive.py --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index dfff8b2c776..d138474e9af 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -72,7 +72,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, f"{arch_dict.RUN}_arcdir.yaml.j2") + arcdir_j2yaml = os.path.join(archive_parm, f"{arch_dict.NET}_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit From c8fd8d2044a85daef9d3bef31e3e7ab28994bfd6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:50:28 -0400 Subject: [PATCH 34/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index eed87658605..e75cb52e5e7 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,7 +31,7 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'IAUFHRS', 'DO_FIT2OBS'] + 'IAUFHRS', 'DO_FIT2OBS', 'NET'] archive_dict = AttrDict() for key in keys: From e324bb510c7c56ebb1d528236140aec5dc5f03ca Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 28 Aug 2024 23:55:59 -0400 Subject: [PATCH 35/67] modified exglobal_archive.py --- parm/archive/arcdir.yaml.j2 | 156 ------------------------------------ 1 file changed, 156 deletions(-) delete mode 100644 parm/archive/arcdir.yaml.j2 diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 deleted file mode 100644 index 57dbc78885e..00000000000 --- a/parm/archive/arcdir.yaml.j2 +++ /dev/null @@ -1,156 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set head = RUN + ".t" + cycle_HH + "z." %} - -# Select data to store in the ARCDIR and VFYARC from deterministic runs -# This file set will contain all source-destination pairs to send to the FileHandler for copying -{% set file_set = [] %} - -# Declare the VFYARC where Fit2Obs data will be sent -{% set VFYARC = ROTDIR ~ "/vrfyarch" %} - -# Deterministic files -{% if "enkf" not in RUN %} - # Common files to be added to both the gfs and gdas keys below - {% set det_files = [] %} - # Cyclone forecasts, produced for both gdas and gfs cycles - ## Only created if tracking is on and there were systems to track - {% if path_exists(COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} - {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH, - ARCDIR ~"/atcfunix." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH, - ARCDIR ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH]) %} - {% endif %} - - # Cyclone tracking data - {% for basin in ["epac", "natl"] %} - {% if path_exists(COMIN_ATMOS_TRACK + "/" + basin) %} - {% do det_files.append([COMIN_ATMOS_TRACK ~ "/" ~ basin, - ARCDIR ~ "/" ~ basin ]) %} - {% endif %} - {% endfor %} - - # Deterministic analysis files (generated for cycled experiments) - {% set det_anl_files = [] %} - # Analysis data (if we are running in cycled mode) - {% do det_anl_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.anl", - ARCDIR ~ "/pgbanl." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} - - {% if DO_JEDIATMVAR == True %} - {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "atmstat", - ARCDIR ~ "/atmstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% else %} - {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "gsistat", - ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% endif %} - - {% if DO_JEDISNOWDA == True %} - {% do det_anl_files.append([COMIN_SNOW_ANALYSIS ~ "/" ~ head ~ "snowstat.tgz", - ARCDIR ~ "/snowstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".tgz"]) %} - {% endif %} - - {% if AERO_ANL_RUN == RUN or AERO_ANL_RUN == "both" %} - {% do det_anl_files.append([COMIN_CHEM_ANALYSIS ~ "/" ~ head ~ "aerostat", - ARCDIR ~ "/aerostat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% endif %} - - {% if DO_PREP_OBS_AERO == True %} - {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroobs", - ARCDIR ~ "/aeroobs." ~ RUN ~ "." ~ cycle_YMDH]) %} - {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroawobs", - ARCDIR ~ "/aeroawobs." ~ RUN ~ "." ~ cycle_YMDH]) %} - {% endif %} - - # GFS-specific files - {% set gfs_files = [] %} - {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} - {% do gfs_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), - ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} - {% endfor %} - - # Cyclone genesis data (only present if there are storms) - {% if path_exists(COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} - {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH, - ARCDIR ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH ]) %} - {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH, - ARCDIR ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH ]) %} - {% endif %} - - {% if path_exists(COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} - {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH, - ARCDIR ~ "/trak.gfso.atcfunix." ~ cycle_YMDH ]) %} - {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH, - ARCDIR ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH ]) %} - {% endif %} - - # GFS Fit2Obs data - {% set fit2obs_files = [] %} - {% for fhr in range(0, FHMAX_FITS + 1, 6) %} - {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} - {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} - {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sfcfile, - VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sfcfile ]) %} - {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sigfile, - VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sigfile ]) %} - {% endfor %} - - # GDAS-specific files - {% set gdas_files = [] %} - {% for fhr in range(0, FHMAX + 1, FHOUT) %} - {% do gdas_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), - ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} - {% endfor %} - - # Now append the necessary file pairs to file_set - # Common deterministic files - {% set file_set = file_set + det_files %} - {% if MODE == "cycled" %} - {% set file_set = file_set + det_anl_files %} - {% endif %} - - # Run-specific deterministic files - {% if RUN == "gfs" %} - {% set file_set = file_set + gfs_files %} - # Fit2Obs files - {% if DO_FIT2OBS == True %} - {% set file_set = file_set + fit2obs_files %} - {% endif %} - {% elif RUN == "gdas" %} - {% set file_set = file_set + gdas_files %} - {% endif %} - -{% else %} # End of deterministic files - - # Ensemble analysis files - {% set enkf_files = [] %} - {% if DO_JEDIATMENS == True %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atmensstat", - ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atminc.ensmean.nc", - ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean.nc"]) %} - {% else %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat", - ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean", - ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} - {% endif %} - - # Construct the final file set - {% set file_set = file_set + enkf_files %} - -{% endif %} - - -# Actually write the yaml -mkdir: - - "{{ ARCDIR }}" - - {% if DO_FIT2OBS == True %} - - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" - {% endif %} - -copy: - {% for source_dest_pair in file_set %} - - {{ source_dest_pair }} - {% endfor %} From dac46ddf7828744dacbb87ca9a603c567ba1cd7f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 00:36:22 -0400 Subject: [PATCH 36/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index e75cb52e5e7..2c6200baf0f 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -40,7 +40,7 @@ def main(): # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config.get[key, "Undefined"] + archive_dict[key] = archive.task_config.get[key] cwd = os.getcwd() From 3d7e3663197dd0a5987175071bba01eea78b4ff6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 00:38:36 -0400 Subject: [PATCH 37/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 2c6200baf0f..e75cb52e5e7 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -40,7 +40,7 @@ def main(): # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config.get[key] + archive_dict[key] = archive.task_config.get[key, "Undefined"] cwd = os.getcwd() From 7cf40fa9d591d3ad170ac9f229381776eae0cc26 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 20:07:12 -0400 Subject: [PATCH 38/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index e75cb52e5e7..24538d0c66d 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -35,12 +35,18 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config.get(key, "Undefined") + try: + archive_dict[key] = archive.task_config.get(key) + except KeyError: + print(f"Key {key} not found") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config.get[key, "Undefined"] + try: + archive_dict[key] = archive.task_config.get[key] + except KeyError: + print(f"Key {key} not found") cwd = os.getcwd() From e938b58b2bde204987660b834a3b1dfcf7305384 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 20:19:35 -0400 Subject: [PATCH 39/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 24538d0c66d..0e6f23e1a02 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -44,7 +44,7 @@ def main(): for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): try: - archive_dict[key] = archive.task_config.get[key] + archive_dict[key] = archive.task_config.get(key) except KeyError: print(f"Key {key} not found") From 1bb9f1391d7fae795c077f86cb7887e1a95c6dc6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 22:29:36 -0400 Subject: [PATCH 40/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 0e6f23e1a02..ef394fc3319 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -37,16 +37,17 @@ def main(): for key in keys: try: archive_dict[key] = archive.task_config.get(key) - except KeyError: - print(f"Key {key} not found") + except: + pass + print(archive_dict) # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): try: archive_dict[key] = archive.task_config.get(key) - except KeyError: - print(f"Key {key} not found") + except: + pass cwd = os.getcwd() From 28f473745ba406a8c5ba0e25587cfbef7fbea960 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 29 Aug 2024 22:31:15 -0400 Subject: [PATCH 41/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index ef394fc3319..082f6e3049a 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -39,7 +39,6 @@ def main(): archive_dict[key] = archive.task_config.get(key) except: pass - print(archive_dict) # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): From e2f2c543bff1263ea9d39a801e4fdaeb9c9f3233 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 30 Aug 2024 12:26:46 -0400 Subject: [PATCH 42/67] modified exglobal_Archive.py --- scripts/exglobal_archive.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 082f6e3049a..22390f9fcf3 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -36,17 +36,19 @@ def main(): archive_dict = AttrDict() for key in keys: try: + archive_dict[key] = archive.task_config(key) + except KeyError: archive_dict[key] = archive.task_config.get(key) - except: - pass + print(f"Key {key} not found in task_config") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): try: + archive_dict[key] = archive.task_config(key) + except KeyError: archive_dict[key] = archive.task_config.get(key) - except: - pass + print(f"Key {key} not found in task_config") cwd = os.getcwd() From 2be37d946ddd5b204b612637c2c63f36987a9e3f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 30 Aug 2024 13:02:13 -0400 Subject: [PATCH 43/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 22390f9fcf3..b6ec4e6bb18 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -36,7 +36,7 @@ def main(): archive_dict = AttrDict() for key in keys: try: - archive_dict[key] = archive.task_config(key) + archive_dict[key] = archive.task_config[key] except KeyError: archive_dict[key] = archive.task_config.get(key) print(f"Key {key} not found in task_config") @@ -45,7 +45,7 @@ def main(): for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): try: - archive_dict[key] = archive.task_config(key) + archive_dict[key] = archive.task_config[key] except KeyError: archive_dict[key] = archive.task_config.get(key) print(f"Key {key} not found in task_config") From 033721eb1ebcaef2984f11d1e75189c8a4fd35c4 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 30 Aug 2024 13:09:00 -0400 Subject: [PATCH 44/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index b6ec4e6bb18..4a0dc1308d9 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -39,7 +39,7 @@ def main(): archive_dict[key] = archive.task_config[key] except KeyError: archive_dict[key] = archive.task_config.get(key) - print(f"Key {key} not found in task_config") + print(f"Key ({key}) not found in task_config") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): @@ -48,7 +48,7 @@ def main(): archive_dict[key] = archive.task_config[key] except KeyError: archive_dict[key] = archive.task_config.get(key) - print(f"Key {key} not found in task_config") + print(f"Key ({key}) not found in task_config") cwd = os.getcwd() From 29a24232edcc8da861fdcf9a5fb644a1ad3a0637 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 11:53:44 -0400 Subject: [PATCH 45/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 4a0dc1308d9..88ccad47727 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,23 +31,19 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'IAUFHRS', 'DO_FIT2OBS', 'NET'] + 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'MEMDIR'] archive_dict = AttrDict() for key in keys: - try: - archive_dict[key] = archive.task_config[key] - except KeyError: - archive_dict[key] = archive.task_config.get(key) + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: print(f"Key ({key}) not found in task_config") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COMIN_") or key.startswith("COMOUT_"): - try: - archive_dict[key] = archive.task_config[key] - except KeyError: - archive_dict[key] = archive.task_config.get(key) + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: print(f"Key ({key}) not found in task_config") cwd = os.getcwd() From 161db4ffc48e06dd21ddc3fd14d19a8b2d6c6e07 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 11:55:50 -0400 Subject: [PATCH 46/67] modify JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index b6e8b952cd4..3a229137ed4 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -51,7 +51,7 @@ for grid in "0p25" "0p50" "1p00"; do done for grid in '0p25' '0p50' '1p00'; do - prod_dir="COMOUT_ATMOS_ENSSTAT_${grid}" + prod_dir="COMIN_ATMOS_ENSSTAT_${grid}" MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" done ############################################################### From ea2166fe60a9f00e9f065127032808d4afcd894f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 11:56:50 -0400 Subject: [PATCH 47/67] modify gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 410852bc4ba..6135dba6770 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -11,9 +11,9 @@ # test files {% if RUN == "gefs" %} {% set ensstat_files = [] %} - {% if path_exists(COMOUT_ATMOS_ENSSTAT_1p00) %} + {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - {% do ensstat_files.append([COMOUT_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", GEFS_ARCH]) %} {% endfor %} From f075b01041d402692573538a90c4957b00804fd4 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 11:57:53 -0400 Subject: [PATCH 48/67] updated exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 88ccad47727..66f9b5eac54 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,7 +31,7 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'MEMDIR'] + 'IAUFHRS', 'DO_FIT2OBS', 'NET'] archive_dict = AttrDict() for key in keys: From 1538f94557a14f363554964053aa2179d7687bf9 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 15:25:37 -0400 Subject: [PATCH 49/67] modified exglobal_archive.py --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 66f9b5eac54..257dfde07f3 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -41,7 +41,7 @@ def main(): # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COMIN_") or key.startswith("COMOUT_"): + if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"): archive_dict[key] = archive.task_config.get(key) if archive_dict[key] is None: print(f"Key ({key}) not found in task_config") From 196cd1b2fe18fb372e7512f3093655670310f7ba Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 15:26:49 -0400 Subject: [PATCH 50/67] modify gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 6135dba6770..9d6fdac8166 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -8,6 +8,15 @@ {% set file_set = [] %} +{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${GRID}': '1p00', + '${MEMDIR}': 'ensstat' }) %} + +{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %} + # test files {% if RUN == "gefs" %} {% set ensstat_files = [] %} From b40fdab43b157b95c2eab2ae8c72de12083f60f0 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 15:31:00 -0400 Subject: [PATCH 51/67] modifed JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 3a229137ed4..aec2a004791 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -49,11 +49,6 @@ for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done - -for grid in '0p25' '0p50' '1p00'; do - prod_dir="COMIN_ATMOS_ENSSTAT_${grid}" - MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" -done ############################################################### # Run archive script ############################################################### From 9119a3c960938656bd7f761e31513028b931e7af Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 15:32:05 -0400 Subject: [PATCH 52/67] modified JGLOBAL_ARCHIVE --- jobs/JGLOBAL_ARCHIVE | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index aec2a004791..401feba35f6 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -49,6 +49,7 @@ for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done + ############################################################### # Run archive script ############################################################### From a71fcbb8eaa3df1e02d75b99326f77c987cf0c22 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 4 Sep 2024 16:12:04 -0400 Subject: [PATCH 53/67] removed arch_test.sh --- jobs/rocoto/arch_test.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 jobs/rocoto/arch_test.sh diff --git a/jobs/rocoto/arch_test.sh b/jobs/rocoto/arch_test.sh deleted file mode 100755 index c723c842aa7..00000000000 --- a/jobs/rocoto/arch_test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/env bash -############################################################### -exit 0 From 55db0a2727bb3cf4d6536247e288be8e92debc58 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 5 Sep 2024 10:47:40 -0400 Subject: [PATCH 54/67] modified exgdas_enkf_earc.py --- scripts/exgdas_enkf_earc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index a515ec97467..f90650a8511 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -28,7 +28,7 @@ def main(): 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', - 'IAUFHRS_ENKF'] + 'IAUFHRS_ENKF', 'NET'] archive_dict = AttrDict() for key in keys: From 283e57faaaf075dc850bb2f5f9411f52c06d8241 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 5 Sep 2024 10:54:43 -0400 Subject: [PATCH 55/67] modified exdas_enkf_earc.py --- scripts/exgdas_enkf_earc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index f90650a8511..33f329f9abf 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -32,7 +32,9 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Key ({key}) not found in task_config") # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): From c3704a56c96343981c20c5dbe20ec6f55a8dcbfe Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 00:28:00 -0400 Subject: [PATCH 56/67] modified config.base --- parm/config/gefs/config.base | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index d601b532cd1..008597bb342 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -330,8 +330,8 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options -export HPSSARCH="@HPSSARCH@" # save data to HPSS archive -export LOCALARCH="@LOCALARCH@" # save data to local archive +export HPSSARCH="NO" # save data to HPSS archive +export LOCALARCH="NO" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 2 From a4550a662a58f28de1ce33848d2b782b858a0cd8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 00:42:55 -0400 Subject: [PATCH 57/67] modified config.base --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 008597bb342..0bbe6600053 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -331,7 +331,7 @@ export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options export HPSSARCH="NO" # save data to HPSS archive -export LOCALARCH="NO" # save data to local archive +export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 2 From 8f26607090cdf212677adaab28abb361037cc8a7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 11:43:06 -0400 Subject: [PATCH 58/67] modified config.base --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 0bbe6600053..008597bb342 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -331,7 +331,7 @@ export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options export HPSSARCH="NO" # save data to HPSS archive -export LOCALARCH="@LOCALARCH@" # save data to local archive +export LOCALARCH="NO" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 2 From 72ae42c043978f26a6f5139e533bb3b32195b966 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 12:22:04 -0400 Subject: [PATCH 59/67] modified config.base and parm/config/gefs/yaml/defaults.yaml --- parm/archive/gefs.yaml.j2 | 10 ++++++++++ parm/config/gefs/config.base | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 parm/archive/gefs.yaml.j2 diff --git a/parm/archive/gefs.yaml.j2 b/parm/archive/gefs.yaml.j2 new file mode 100644 index 00000000000..7330c1296b0 --- /dev/null +++ b/parm/archive/gefs.yaml.j2 @@ -0,0 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} + +gefs: + name: "GEFS" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs.tar" + required: + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/gefs.t12z.spread.pres_.1p00.f120.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/gefs.t12z.spread.pres_.1p00.f120.grib2" diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 008597bb342..d601b532cd1 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -330,8 +330,8 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options -export HPSSARCH="NO" # save data to HPSS archive -export LOCALARCH="NO" # save data to local archive +export HPSSARCH="@HPSSARCH@" # save data to HPSS archive +export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 2 From 5b971b8db45194ba494d807b456cc027168c80b8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 12:26:55 -0400 Subject: [PATCH 60/67] modified config.base --- parm/config/gefs/config.base | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 008597bb342..d601b532cd1 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -330,8 +330,8 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options -export HPSSARCH="NO" # save data to HPSS archive -export LOCALARCH="NO" # save data to local archive +export HPSSARCH="@HPSSARCH@" # save data to HPSS archive +export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 2 From 7fad1f1afb6e1d51f1d5c5a55d38fee79eec9a7a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 12:36:19 -0400 Subject: [PATCH 61/67] deleted gefs.yaml --- parm/archive/gefs.yaml.j2 | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 parm/archive/gefs.yaml.j2 diff --git a/parm/archive/gefs.yaml.j2 b/parm/archive/gefs.yaml.j2 deleted file mode 100644 index 7330c1296b0..00000000000 --- a/parm/archive/gefs.yaml.j2 +++ /dev/null @@ -1,10 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} - -gefs: - name: "GEFS" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs.tar" - required: - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/gefs.t12z.spread.pres_.1p00.f120.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/gefs.t12z.spread.pres_.1p00.f120.grib2" From 48b1eae72fa0695b84f475766bf142fa7e8b5af7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 12:38:11 -0400 Subject: [PATCH 62/67] modified parm/config/gefs/yaml/defaults.yaml --- parm/config/gefs/yaml/defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index e4666d1aba3..3462ef43ee0 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -14,3 +14,5 @@ base: FCST_BREAKPOINTS: "48" REPLAY_ICS: "NO" USE_OCN_PERTURB_FILES: "false" + HPSSARCH: "NO" + LOCALARCH: "NO" \ No newline at end of file From 16801915b5de63c700b2da0a3c07e1cc5099ba4d Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 12:39:16 -0400 Subject: [PATCH 63/67] modifed parm/config/gefs/yaml/defaults.yaml --- parm/config/gefs/yaml/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index 3462ef43ee0..5ecf690e180 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -15,4 +15,4 @@ base: REPLAY_ICS: "NO" USE_OCN_PERTURB_FILES: "false" HPSSARCH: "NO" - LOCALARCH: "NO" \ No newline at end of file + LOCALARCH: "NO" From abee6045f71f36b61370247977a7f71bcbbfde2f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 13:06:59 -0400 Subject: [PATCH 64/67] added a test to archive.py --- ush/python/pygfs/task/archive.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index d138474e9af..14cd0156017 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -114,6 +114,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.tar_cmd = "" return arcdir_set, [] + if arch_dict.NET == "gefs": + raise NotImplementedError("GEFS archiving is not yet implemented!") + master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), From b82e1d5b1a46b74f742e2367336e7b4b2f9b806e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 13:09:50 -0400 Subject: [PATCH 65/67] modified exglobal_Archive.py --- scripts/exglobal_archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 257dfde07f3..793fa1c1ac4 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -37,14 +37,14 @@ def main(): for key in keys: archive_dict[key] = archive.task_config.get(key) if archive_dict[key] is None: - print(f"Key ({key}) not found in task_config") + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"): archive_dict[key] = archive.task_config.get(key) if archive_dict[key] is None: - print(f"Key ({key}) not found in task_config") + print(f"Warning: key ({key}) not found in task_config!") cwd = os.getcwd() From 692de265d46d1fc901fe132c80bac157dc4d766f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 13:21:39 -0400 Subject: [PATCH 66/67] modified the warning message exgdas_enkf_earc.py --- scripts/exgdas_enkf_earc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 33f329f9abf..c724bdbd673 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -34,7 +34,7 @@ def main(): for key in keys: archive_dict[key] = archive.task_config.get(key) if archive_dict[key] is None: - print(f"Key ({key}) not found in task_config") + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): From 18bc47089aa5b269792d1d55248ab9e24520bb6f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 6 Sep 2024 13:32:25 -0400 Subject: [PATCH 67/67] modified gefs_arcdir.yaml.j2 --- parm/archive/gefs_arcdir.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 9d6fdac8166..a59a0e1a8f5 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -17,7 +17,7 @@ {% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %} -# test files +# Select ensstat files to copy to the arcdir {% if RUN == "gefs" %} {% set ensstat_files = [] %} {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %}