From 4494db30b20dba8f64d9e7a93d8601db635df475 Mon Sep 17 00:00:00 2001 From: "kayee.wong" Date: Thu, 6 Mar 2025 17:06:20 +0000 Subject: [PATCH 1/2] Update realtime branch, gsl_ufs_rt --- .flake8 | 3 + .github/CODEOWNERS | 214 ++ .github/ISSUE_TEMPLATE/NCO_bug_report.yml | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 14 +- .github/ISSUE_TEMPLATE/dump_request.yml | 37 + .github/ISSUE_TEMPLATE/feature_request.yml | 17 +- .github/ISSUE_TEMPLATE/production_update.yml | 29 +- .github/ISSUE_TEMPLATE/static_data.yml | 134 + .github/pull_request_template.md | 64 +- .github/workflows/ci_unit_tests.yaml | 4 +- .github/workflows/docs.yaml | 6 +- .github/workflows/hera.yaml | 2 +- .github/workflows/hercules.yaml | 81 + .github/workflows/linters.yaml | 4 +- .github/workflows/pw_aws_ci.yaml | 173 ++ .github/workflows/pynorms.yaml | 2 +- .github/workflows/wcoss2.yaml | 81 + .gitignore | 59 +- CMakeLists.txt | 27 + FV3GFSwfm/rt_c384ics/config.analcalc | 15 - FV3GFSwfm/rt_c384ics/config.eobs | 31 - FV3GFSwfm/rt_c384ics/config.epos | 20 - FV3GFSwfm/rt_c384ics/config.esfc | 19 - FV3GFSwfm/rt_c384ics/config.eupd | 34 - FV3GFSwfm/rt_c384ics/config.wavepostsbs | 28 - INFO | 13 +- README.md | 44 +- ci/Jenkinsfile | 340 ++- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 12 +- ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml | 20 + ci/cases/gfsv17/marine3dvar.yaml | 27 + ci/cases/gfsv17/marinehyb.yaml | 25 + ci/cases/pr/C48_S2SWA_gefs.yaml | 6 +- ci/cases/pr/C48_S2SW_extended.yaml | 21 + ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 12 +- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 24 + ci/cases/pr/C96C48_hybatmDA.yaml | 4 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 24 + ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 10 +- ci/cases/pr/C96_atm3DVar.yaml | 6 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 5 +- ci/cases/pr/C96mx100_S2S.yaml | 19 + ci/cases/sfs/C96mx100_S2S.yaml | 19 + ci/cases/weekly/C384C192_hybatmda.yaml | 4 +- ci/cases/weekly/C384_atm3DVar.yaml | 4 +- ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml | 1 + ci/cases/yamls/gefs_defaults_ci.yaml | 11 + ci/cases/yamls/gefs_replay_ci.yaml | 18 + ci/cases/yamls/gfs_defaults_ci.yaml | 4 + ci/cases/yamls/gfs_extended_ci.yaml | 5 +- ci/cases/yamls/gfs_metp_ci.yaml | 6 + ci/cases/yamls/sfs_defaults.yaml | 31 + ci/cases/yamls/sfs_full.yaml | 31 + ci/cases/yamls/soca_gfs_defaults_ci.yaml | 7 + ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml | 8 + ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 8 +- ci/platforms/config.gaeac5 | 7 + ci/platforms/config.gaeac6 | 7 + ci/platforms/config.hera | 2 + ci/platforms/config.hercules | 2 +- ci/platforms/config.orion | 1 + ci/scripts/check_ci.sh | 13 +- ci/scripts/driver.sh | 13 +- ci/scripts/driver_weekly.sh | 2 +- ci/scripts/run_ci.sh | 2 +- ci/scripts/tests/test_setup.py | 2 +- ci/scripts/utils/ci_utils.sh | 26 +- ci/scripts/utils/ci_utils_wrapper.sh | 4 +- ci/scripts/utils/get_config_var.py | 43 + ci/scripts/utils/launch_java_agent.sh | 19 +- .../parallel_works/UserBootstrap_centos7.txt | 5 + .../utils/parallel_works/provision_runner.sh | 39 + ci/scripts/utils/rocotostat.py | 15 +- ci/scripts/wxflow | 1 + ctests/CMakeLists.txt | 118 + ctests/README.md | 70 + ctests/cases/C48_ATM_gfs_fcst_seg0.yaml | 157 ++ .../C48_S2SW_gfs_atmos_prod_f000-f003.yaml | 136 + ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml | 50 + ctests/scripts/execute.sh.in | 66 + ctests/scripts/setup.sh.in | 31 + ctests/scripts/stage.py | 70 + ctests/scripts/stage.sh.in | 36 + ctests/scripts/validate.py | 120 + ctests/scripts/validate.sh.in | 36 + docs/note_fixfield.txt | 2 +- docs/source/clone.rst | 66 +- docs/source/components.rst | 3 +- docs/source/conf.py | 4 +- docs/source/configure.rst | 13 +- docs/source/development.rst | 25 +- docs/source/errors_faq.rst | 9 +- docs/source/hpc.rst | 191 +- docs/source/index.rst | 9 +- docs/source/init.rst | 16 +- docs/source/jobs.rst | 8 +- docs/source/noaa_csp.rst | 458 +++- docs/source/references.bib | 0 docs/source/run.rst | 1 - docs/source/setup.rst | 25 +- docs/source/wave.rst | 8 +- env/AWSPW.env | 76 +- env/AZUREPW.env | 108 + env/CONTAINER.env | 4 +- env/GAEAC5.env | 295 +++ env/GAEAC6.env | 293 +++ env/GOOGLEPW.env | 107 + env/HERA.env | 121 +- env/HERCULES.env | 104 +- env/JET.env | 74 +- env/ORION.env | 96 +- env/S4.env | 70 +- env/WCOSS2.env | 100 +- gempak/ush/gdas_ecmwf_meta_ver.sh | 8 +- gempak/ush/gdas_meta_loop.sh | 12 +- gempak/ush/gdas_meta_na.sh | 8 +- gempak/ush/gdas_ukmet_meta_ver.sh | 8 +- gempak/ush/gempak_gdas_f000_gif.sh | 44 +- gempak/ush/gempak_gfs_f000_gif.sh | 66 +- gempak/ush/gempak_gfs_fhhh_gif.sh | 18 +- gempak/ush/gfs_meta_comp.sh | 2 +- gempak/ush/gfs_meta_mar_comp.sh | 2 +- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 46 + jobs/JGDAS_ATMOS_ANALYSIS_DIAG | 5 +- jobs/JGDAS_ATMOS_GEMPAK_META_NCDC | 14 +- jobs/JGDAS_ENKF_ARCHIVE_TARS | 44 + jobs/JGDAS_ENKF_ARCHIVE_VRFY | 43 + jobs/JGDAS_ENKF_DIAG | 48 +- jobs/JGDAS_ENKF_ECEN | 6 +- jobs/JGDAS_ENKF_SELECT_OBS | 64 +- jobs/JGDAS_ENKF_UPDATE | 9 +- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN | 16 +- jobs/JGFS_ATMOS_CYCLONE_GENESIS | 5 - jobs/JGFS_ATMOS_CYCLONE_TRACKER | 5 - jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF | 13 +- jobs/JGFS_ATMOS_VERIFICATION | 6 - jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 +- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +- jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL | 35 + jobs/JGLOBAL_ARCHIVE_TARS | 95 + jobs/JGLOBAL_ARCHIVE_VRFY | 50 + jobs/JGLOBAL_ATMENS_ANALYSIS_OBS | 35 + jobs/JGLOBAL_ATMENS_ANALYSIS_SOL | 35 + jobs/JGLOBAL_ATMOS_ANALYSIS | 30 +- jobs/JGLOBAL_ATMOS_ANALYSIS_CALC | 13 +- jobs/JGLOBAL_ATMOS_UPP | 7 +- jobs/JGLOBAL_ATMOS_VMINMON | 18 +- jobs/JGLOBAL_EXTRACTVARS | 5 + jobs/JGLOBAL_FETCH | 23 + jobs/JGLOBAL_FORECAST | 15 +- jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT | 37 + jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE | 44 + jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE | 55 + jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 25 +- jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL | 39 + jobs/JGLOBAL_MARINE_BMAT | 16 +- jobs/JGLOBAL_SNOWENS_ANALYSIS | 70 + jobs/JGLOBAL_SNOW_ANALYSIS | 16 +- jobs/JGLOBAL_STAGE_IC | 17 +- jobs/JGLOBAL_WAVE_GEMPAK | 16 +- jobs/JGLOBAL_WAVE_POST_PNT | 2 +- jobs/JGLOBAL_WAVE_POST_SBS | 21 +- jobs/JGLOBAL_WAVE_PRDGEN_GRIDDED | 12 +- jobs/rocoto/aeroanlgenb.sh | 19 + jobs/rocoto/aeroanlvar.sh | 18 + jobs/rocoto/{arch.sh => arch_tars.sh} | 4 +- jobs/rocoto/arch_vrfy.sh | 24 + jobs/rocoto/atmensanlobs.sh | 18 + jobs/rocoto/atmensanlsol.sh | 18 + jobs/rocoto/atmos_ensstat.sh | 21 +- jobs/rocoto/atmos_products.sh.emc | 36 + jobs/rocoto/{earc.sh => earc_tars.sh} | 4 +- jobs/rocoto/earc_vrfy.sh | 24 + jobs/rocoto/esnowanl.sh | 26 + jobs/rocoto/fetch.sh | 18 + jobs/rocoto/gempak.sh | 19 +- jobs/rocoto/init.sh | 2 +- jobs/rocoto/marineanlchkpt.sh | 18 + jobs/rocoto/marineanlfinal.sh | 18 + jobs/rocoto/marineanlinit.sh | 19 + jobs/rocoto/marineanlletkf.sh | 23 + jobs/rocoto/marineanlvar.sh | 18 + jobs/rocoto/oceanice_products.sh | 21 +- jobs/rocoto/prepoceanobs.sh | 6 +- jobs/rocoto/snowanl.sh | 8 + jobs/rocoto/upp.sh | 2 +- jobs/rocoto/wavepostsbs.sh | 21 +- modulefiles/module_base.gaeac5.lua | 48 + modulefiles/module_base.gaeac6.lua | 47 + modulefiles/module_base.hera.lua | 2 + modulefiles/module_base.hercules.lua | 6 +- modulefiles/module_base.jet.lua | 4 +- modulefiles/module_base.noaacloud.lua | 52 + modulefiles/module_base.orion.lua | 6 +- modulefiles/module_base.wcoss2.lua | 3 +- modulefiles/module_gwci.noaacloud.lua | 15 + modulefiles/module_gwsetup.gaeac5.lua | 20 + modulefiles/module_gwsetup.gaeac6.lua | 20 + modulefiles/module_gwsetup.hercules.lua | 1 + modulefiles/module_gwsetup.jet.lua | 2 +- modulefiles/module_gwsetup.noaacloud.lua | 21 + modulefiles/module_gwsetup.orion.lua | 2 + modulefiles/module_gwsetup.wcoss2.lua | 3 + parm/archive/enkf.yaml.j2 | 61 +- parm/archive/enkf_grp.yaml.j2 | 2 + parm/archive/enkf_restarta_grp.yaml.j2 | 4 +- parm/archive/enkf_restartb_grp.yaml.j2 | 7 +- parm/archive/expdir.yaml.j2 | 24 + parm/archive/gdas.yaml.j2 | 62 +- parm/archive/gdas_restarta.yaml.j2 | 6 +- parm/archive/gdas_restartb.yaml.j2 | 5 +- parm/archive/gdaswave.yaml.j2 | 14 +- parm/archive/gefs_arcdir.yaml.j2 | 42 + parm/archive/gfs_arcdir.yaml.j2 | 156 ++ parm/archive/gfs_downstream.yaml.j2 | 6 +- parm/archive/gfs_netcdfa.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 9 +- parm/archive/gfswave.yaml.j2 | 20 +- parm/archive/master_enkf.yaml.j2 | 22 - parm/archive/master_gdas.yaml.j2 | 19 +- parm/archive/master_gefs.yaml.j2 | 12 + parm/archive/master_gfs.yaml.j2 | 21 +- parm/config/gefs/config.arch_tars | 15 + parm/config/gefs/config.arch_vrfy | 15 + parm/config/gefs/config.atmos_ensstat | 3 + parm/config/gefs/config.atmos_products | 19 +- parm/config/gefs/config.base | 110 +- parm/config/gefs/config.cleanup | 1 + parm/config/gefs/config.efcs | 50 +- parm/config/gefs/config.extractvars | 10 +- parm/config/gefs/config.fcst | 77 +- parm/config/gefs/config.fetch | 1 + parm/config/gefs/config.nsst | 4 - parm/config/gefs/config.oceanice_products | 4 +- parm/config/gefs/config.resources | 126 +- parm/config/gefs/config.resources.AWSPW | 69 + parm/config/gefs/config.resources.AZUREPW | 11 + parm/config/gefs/config.resources.GOOGLEPW | 11 + parm/config/gefs/config.stage_ic | 55 +- parm/config/gefs/config.ufs | 129 +- parm/config/gefs/config.wave | 21 +- parm/config/gefs/config.wavepostsbs | 3 + parm/config/gefs/yaml/defaults.yaml | 21 +- parm/config/gfs/config.aero | 30 +- parm/config/gfs/config.aeroanl | 32 +- parm/config/gfs/config.aeroanlgenb | 27 + parm/config/gfs/config.aeroanlvar | 11 + parm/config/gfs/config.anal | 5 + parm/config/gfs/config.arch_tars | 15 + parm/config/gfs/config.arch_vrfy | 15 + parm/config/gfs/config.atmanl | 8 +- parm/config/gfs/config.atmanlfv3inc | 3 - parm/config/gfs/config.atmensanl | 10 +- parm/config/gfs/config.atmensanlfv3inc | 3 - parm/config/gfs/config.atmensanlobs | 11 + parm/config/gfs/config.atmensanlsol | 11 + parm/config/gfs/config.atmos_products | 6 +- parm/config/gfs/config.base | 520 +++- parm/config/gfs/config.base.hera | 3 - .../gfs/config.base~gsl_ufs_dev_Mar05_af805b | 519 ++++ parm/config/gfs/config.com | 38 +- parm/config/gfs/config.earc_tars | 35 + .../config/gfs/config.earc_vrfy | 12 +- parm/config/gfs/config.efcs | 2 +- parm/config/gfs/config.esnowanl | 38 + parm/config/gfs/config.eupd | 2 +- parm/config/gfs/config.fcst | 50 +- parm/config/gfs/config.fetch | 19 + parm/config/gfs/config.gempak | 5 +- parm/config/gfs/config.marineanl | 24 + parm/config/gfs/config.marineanlchkpt | 11 + parm/config/gfs/config.marineanlfinal | 10 + parm/config/gfs/config.marineanlinit | 10 + parm/config/gfs/config.marineanlletkf | 20 + parm/config/gfs/config.marineanlvar | 11 + parm/config/gfs/config.oceanice_products | 3 + parm/config/gfs/config.ocn | 9 +- parm/config/gfs/config.prepoceanobs | 9 +- parm/config/gfs/config.resources | 424 +++- parm/config/gfs/config.resources.AWSPW | 35 + parm/config/gfs/config.resources.AZUREPW | 11 + parm/config/gfs/config.resources.GAEAC5 | 8 + parm/config/gfs/config.resources.GAEAC6 | 6 + parm/config/gfs/config.resources.GOOGLEPW | 11 + parm/config/gfs/config.resources.HERA | 36 +- parm/config/gfs/config.resources.HERCULES | 11 + parm/config/gfs/config.resources.JET | 7 +- parm/config/gfs/config.resources.ORION | 29 +- parm/config/gfs/config.resources.S4 | 8 +- parm/config/gfs/config.resources.WCOSS2 | 20 +- parm/config/gfs/config.snowanl | 22 +- parm/config/gfs/config.stage_ic | 69 +- parm/config/gfs/config.ufs | 708 +++++- .../gfs/config.ufs~develop_Feb212025_ba09498 | 702 ++++++ .../gfs/config.ufs~gsl_ufs_dev_Mar05_af805b | 707 ++++++ parm/config/gfs/config.wave | 35 +- parm/config/gfs/config.waveawipsbulls | 2 - parm/config/gfs/config.waveawipsgridded | 10 +- parm/config/gfs/config.wavegempak | 2 + parm/config/gfs/config.wavepostsbs | 3 + parm/config/gfs/yaml/defaults.yaml | 24 +- parm/config/sfs/config.aero | 1 + parm/config/sfs/config.arch_tars | 1 + parm/config/sfs/config.arch_vrfy | 1 + parm/config/sfs/config.atmos_ensstat | 1 + parm/config/sfs/config.atmos_products | 1 + parm/config/sfs/config.base | 357 +++ parm/config/sfs/config.cleanup | 1 + parm/config/sfs/config.com | 1 + parm/config/sfs/config.efcs | 1 + parm/config/sfs/config.extractvars | 1 + parm/config/sfs/config.fcst | 1 + parm/config/sfs/config.ice | 1 + parm/config/sfs/config.nsst | 1 + parm/config/sfs/config.oceanice_products | 1 + parm/config/sfs/config.ocn | 1 + parm/config/sfs/config.prep_emissions | 1 + parm/config/sfs/config.resources | 371 +++ parm/config/sfs/config.resources.AWSPW | 1 + parm/config/sfs/config.resources.AZUREPW | 1 + parm/config/sfs/config.resources.GOOGLEPW | 1 + parm/config/sfs/config.stage_ic | 1 + parm/config/sfs/config.ufs | 1 + parm/config/sfs/config.wave | 1 + parm/config/sfs/config.waveinit | 1 + parm/config/sfs/config.wavepostbndpnt | 1 + parm/config/sfs/config.wavepostbndpntbll | 1 + parm/config/sfs/config.wavepostpnt | 1 + parm/config/sfs/config.wavepostsbs | 1 + parm/config/sfs/yaml/defaults.yaml | 34 + parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 | 16 + .../fetch/gfs_S2SW_cold_forecast-only.yaml.j2 | 37 + parm/gdas/aero_bmat_jedi_config.yaml.j2 | 19 + parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 + parm/gdas/aero_finalize_variational.yaml.j2 | 24 + parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 45 + parm/gdas/aero_stage_variational.yaml.j2 | 50 + parm/gdas/aeroanl_jedi_config.yaml.j2 | 7 + parm/gdas/atmanl_jedi_config.yaml.j2 | 13 + parm/gdas/atmensanl_jedi_config.yaml.j2 | 27 + parm/gdas/esnowanl_jedi_config.yaml.j2 | 14 + parm/gdas/snow_finalize_ens_update.yaml.j2 | 43 + parm/gdas/snow_stage_ens_update.yaml.j2 | 52 + parm/gdas/snow_stage_orog.yaml.j2 | 8 + parm/gdas/snowanl_jedi_config.yaml.j2 | 7 + parm/gdas/soca_bmat_jedi_config.yaml.j2 | 49 + parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 27 +- parm/gdas/staging/atm_var_bkg.yaml.j2 | 13 +- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 27 +- parm/gdas/staging/snow_berror.yaml.j2 | 4 + parm/gdas/staging/snow_var_bkg.yaml.j2 | 8 + parm/post/oceanice_products_gefs.yaml | 16 +- parm/post/upp.yaml | 36 +- parm/product/bufr_ij_gfs_C1152.txt | 2236 +++++++++++++++++ .../{bufr_ij13km.txt => bufr_ij_gfs_C768.txt} | 127 +- parm/product/bufr_ij_gfs_C96.txt | 2236 +++++++++++++++++ parm/product/bufr_stalist.meteo.gfs | 127 +- parm/product/gefs.0p25.fFFF.paramlist.a.txt | 7 +- parm/product/gefs.0p25.fFFF.paramlist.b.txt | 96 +- parm/product/sfs.0p25.f000.paramlist.a.txt | 39 + parm/product/sfs.0p25.fFFF.paramlist.a.txt | 52 + parm/product/sfs.0p25.fFFF.paramlist.b.txt | 549 ++++ parm/stage/analysis.yaml.j2 | 19 + parm/stage/atmosphere_cold.yaml.j2 | 18 + .../atmosphere_ens_perturbations.yaml.j2 | 13 + parm/stage/atmosphere_nest.yaml.j2 | 36 + parm/stage/atmosphere_warm.yaml.j2 | 31 + parm/stage/ice.yaml.j2 | 35 + parm/stage/master_gefs.yaml.j2 | 154 ++ parm/stage/master_gfs.yaml.j2 | 184 ++ parm/stage/ocean.yaml.j2 | 18 + parm/stage/ocean_ens_perturbations.yaml.j2 | 13 + parm/stage/ocean_mediator.yaml.j2 | 15 + parm/stage/ocean_rerun.yaml.j2 | 13 + parm/stage/wave.yaml.j2 | 17 + parm/transfer/transfer_gdas_1c.list | 67 - parm/transfer/transfer_gdas_enkf_enkf_05.list | 61 - parm/transfer/transfer_gdas_enkf_enkf_10.list | 61 - parm/transfer/transfer_gdas_enkf_enkf_15.list | 61 - parm/transfer/transfer_gfs_gdas_gdas_1a.list | 19 +- parm/transfer/transfer_gfs_gdas_gdas_1b.list | 22 - parm/transfer/transfer_gfs_gfs_1.list | 14 - parm/transfer/transfer_gfs_gfs_10a.list | 13 - parm/transfer/transfer_gfs_gfs_9a.list | 16 - parm/transfer/transfer_gfs_gfs_gempak.list | 21 +- .../transfer_rdhpcs_gfs_gdas_enkf_enkf_1.list | 20 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_2.list | 20 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_3.list | 8 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_4.list | 20 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_5.list | 18 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_6.list | 18 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_7.list | 8 - .../transfer_rdhpcs_gfs_gdas_enkf_enkf_8.list | 18 - .../transfer_rdhpcs_gfs_gdas_gdas.list | 8 - parm/transfer/transfer_rdhpcs_gfs_gempak.list | 3 - parm/transfer/transfer_rdhpcs_gfs_gfs.list | 4 - parm/ufs/fv3/diag_table | 4 + parm/ufs/fv3/diag_table_da | 1 + parm/ufs/fv3/diag_table_replay | 340 +++ parm/ufs/gocart/AERO_HISTORY.rc | 32 +- parm/ufs/gocart/ExtData.other | 20 +- parm/wave/ak_10m_interp.inp.tmpl | 5 +- parm/wave/at_10m_interp.inp.tmpl | 2 +- parm/wave/bull_awips_gfswave | 2 +- parm/wave/ep_10m_interp.inp.tmpl | 2 +- parm/wave/glo_15mxt_interp.inp.tmpl | 2 +- parm/wave/glo_30m_interp.inp.tmpl | 2 +- parm/wave/wave_gfs.buoys.dat | 2 +- parm/wave/wave_gfs.buoys.full | 4 +- parm/wave/wc_10m_interp.inp.tmpl | 2 +- .../exgdas_aero_analysis_generate_bmatrix.py | 25 + scripts/exgdas_atmos_gempak_gif_ncdc.sh | 2 +- scripts/exgdas_enkf_earc_tars.py | 59 + scripts/exgdas_enkf_earc_vrfy.py | 51 + scripts/exgdas_enkf_ecen.sh | 44 +- scripts/exgdas_enkf_post.sh | 25 +- scripts/exgdas_enkf_sfc.sh | 112 +- scripts/exgdas_enkf_update.sh | 49 +- scripts/exgfs_aero_init_aerosol.py | 20 +- scripts/exgfs_atmos_awips_20km_1p0deg.sh | 10 - scripts/exgfs_atmos_gempak_gif_ncdc_skew_t.sh | 12 +- scripts/exgfs_atmos_postsnd.sh | 140 +- scripts/exgfs_wave_init.sh | 2 +- scripts/exgfs_wave_nawips.sh | 22 +- scripts/exgfs_wave_post_gridded_sbs.sh | 304 +-- scripts/exgfs_wave_post_pnt.sh | 34 +- scripts/exgfs_wave_prdgen_bulls.sh | 12 +- scripts/exgfs_wave_prdgen_gridded.sh | 158 +- scripts/exgfs_wave_prep.sh | 24 +- scripts/exglobal_aero_analysis_finalize.py | 2 + scripts/exglobal_aero_analysis_initialize.py | 2 + scripts/exglobal_aero_analysis_variational.py | 24 + scripts/exglobal_archive_tars.py | 71 + scripts/exglobal_archive_vrfy.py | 58 + scripts/exglobal_atm_analysis_finalize.py | 2 + .../exglobal_atm_analysis_fv3_increment.py | 11 +- scripts/exglobal_atm_analysis_initialize.py | 6 +- scripts/exglobal_atm_analysis_variational.py | 8 +- scripts/exglobal_atmens_analysis_finalize.py | 2 + .../exglobal_atmens_analysis_fv3_increment.py | 11 +- .../exglobal_atmens_analysis_initialize.py | 6 +- scripts/exglobal_atmens_analysis_letkf.py | 11 +- scripts/exglobal_atmens_analysis_obs.py | 24 + scripts/exglobal_atmens_analysis_sol.py | 24 + scripts/exglobal_atmos_analysis.sh | 223 +- scripts/exglobal_atmos_analysis_calc.sh | 48 +- scripts/exglobal_atmos_products.sh | 2 +- scripts/exglobal_atmos_upp.py | 2 +- scripts/exglobal_cleanup.sh | 25 +- scripts/exglobal_diag.sh | 10 +- scripts/exglobal_fetch.py | 39 + .../exglobal_marine_analysis_checkpoint.py | 28 + scripts/exglobal_marine_analysis_finalize.py | 27 + .../exglobal_marine_analysis_initialize.py | 24 + scripts/exglobal_marine_analysis_letkf.py | 24 + .../exglobal_marine_analysis_variational.py | 24 + scripts/exglobal_snow_analysis.py | 21 +- scripts/exglobal_snowens_analysis.py | 43 + scripts/exglobal_stage_ic.py | 46 + sorc/build_all.sh | 208 +- sorc/build_compute.sh | 131 + sorc/build_ufs.sh | 47 +- sorc/build_upp.sh | 25 + sorc/build_ww3prepost.sh | 132 +- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/link_workflow.sh | 291 +-- sorc/ncl.setup | 12 - sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- sorc/verif-global.fd | 2 +- sorc/wxflow | 2 +- ush/atmos_extractvars.sh | 25 +- ush/calcanl_gfs.py | 7 +- ush/calcinc_gfs.py | 2 +- ush/detect_machine.sh | 18 +- ush/extractvars_tools.sh | 3 +- ush/forecast_det.sh | 16 +- ush/forecast_postdet.sh | 562 +++-- ush/forecast_predet.sh | 258 +- ush/fv3gfs_remap.sh | 118 - ush/gaussian_sfcanl.sh | 19 +- ush/gfs_bufr.sh | 92 +- ush/hpssarch_gen.sh | 1 - ush/interp_atmos_master.sh | 6 +- ush/load_fv3gfs_modules.sh | 3 +- ush/load_ufsda_modules.sh | 11 +- ush/load_ufswm_modules.sh | 45 +- ush/make_NTC_file.pl | 118 + ush/make_ntc_bull.pl | 242 ++ ush/make_tif.sh | 31 + ush/module-setup.sh | 8 +- ush/month_name.sh | 113 + ush/ocnice_extractvars.sh | 14 +- ush/parsing_model_configure_FV3.sh | 22 +- ush/parsing_namelists_CICE.sh | 3 +- ush/parsing_namelists_FV3.sh | 119 +- ush/parsing_namelists_FV3_nest.sh | 1 + ush/parsing_namelists_GOCART.sh | 52 + ush/parsing_namelists_MOM6.sh | 6 + ush/parsing_namelists_WW3.sh | 325 +-- ush/parsing_ufs_configure.sh | 22 +- ush/python/pygfs/__init__.py | 3 + ush/python/pygfs/jedi/__init__.py | 1 + ush/python/pygfs/jedi/jedi.py | 352 +++ ush/python/pygfs/task/aero_analysis.py | 323 +-- ush/python/pygfs/task/aero_bmatrix.py | 144 ++ ush/python/pygfs/task/aero_prepobs.py | 2 +- ush/python/pygfs/task/analysis.py | 49 +- ush/python/pygfs/task/archive.py | 238 +- ush/python/pygfs/task/atm_analysis.py | 246 +- ush/python/pygfs/task/atmens_analysis.py | 182 +- ush/python/pygfs/task/fetch.py | 105 + ush/python/pygfs/task/marine_analysis.py | 500 ++++ ush/python/pygfs/task/marine_bmat.py | 313 +-- ush/python/pygfs/task/marine_letkf.py | 78 +- ush/python/pygfs/task/oceanice_products.py | 16 +- ush/python/pygfs/task/snow_analysis.py | 563 ++--- ush/python/pygfs/task/snowens_analysis.py | 447 ++++ ush/python/pygfs/task/stage_ic.py | 64 + ush/python/pygfs/task/upp.py | 11 +- ush/python/pygfs/utils/marine_da_utils.py | 185 +- ush/python/wxflow | 1 + ush/wave_domain_grid.sh | 43 + ush/wave_extractvars.sh | 7 +- ush/wave_grib2_sbs.sh | 91 +- ush/wave_grid_interp_sbs.sh | 48 +- ush/wave_grid_moddef.sh | 21 +- ush/wave_outp_spec.sh | 14 +- ush/wave_prnc_cur.sh | 12 +- ush/wave_prnc_ice.sh | 20 +- versions/build.gaeac5.ver | 5 + versions/build.gaeac6.ver | 5 + versions/build.hera.ver | 2 +- versions/build.hercules.ver | 2 +- versions/build.jet.ver | 6 +- versions/build.noaacloud.ver | 5 + versions/build.orion.ver | 2 +- versions/build.s4.ver | 2 +- versions/fix.ver | 12 +- versions/ic.ver | 22 + versions/run.gaeac5.ver | 8 + versions/run.gaeac6.ver | 8 + versions/run.hera.ver | 4 +- versions/run.hercules.ver | 3 +- versions/run.jet.ver | 10 +- versions/run.noaacloud.ver | 8 + versions/run.orion.ver | 7 +- versions/run.s4.ver | 2 +- versions/run.wcoss2.ver | 7 +- versions/spack.ver | 47 + workflow/applications/application_factory.py | 2 + workflow/applications/applications.py | 212 +- workflow/applications/gefs.py | 62 +- workflow/applications/gfs_cycled.py | 537 ++-- workflow/applications/gfs_forecast_only.py | 134 +- workflow/applications/sfs.py | 104 + workflow/build_compute.py | 178 ++ workflow/build_opts.yaml | 82 + workflow/generate_workflows.sh | 546 ++++ workflow/gsl_template_hera.xml | 2 +- workflow/hosts.py | 11 +- workflow/hosts/awspw.yaml | 28 +- workflow/hosts/azurepw.yaml | 31 + workflow/hosts/gaeac5.yaml | 28 + workflow/hosts/gaeac6.yaml | 27 + workflow/hosts/googlepw.yaml | 30 + workflow/hosts/hera.yaml | 30 +- .../hosts/hera.yaml~develop_Feb212025_ba09498 | 32 + .../hosts/hera.yaml~gsl_ufs_dev_Mar05_af805b | 29 + workflow/hosts/hercules.yaml | 7 +- workflow/hosts/jet.yaml | 29 +- .../hosts/jet.yaml~develop_Feb212025_ba09498 | 31 + .../hosts/jet.yaml~gsl_ufs_dev_Mar05_af805b | 28 + workflow/hosts/orion.yaml | 9 +- workflow/hosts/s4.yaml | 5 +- workflow/hosts/wcoss2.yaml | 4 +- workflow/mynn.sh | 18 + workflow/rocoto/gefs_tasks.py | 507 ++-- workflow/rocoto/gefs_xml.py | 14 +- workflow/rocoto/gfs_cycled_xml.py | 33 +- workflow/rocoto/gfs_forecast_only_xml.py | 37 +- workflow/rocoto/gfs_tasks.py | 1260 ++++++---- workflow/rocoto/rocoto.py | 25 +- workflow/rocoto/rocoto_xml_factory.py | 3 +- workflow/rocoto/sfs_tasks.py | 656 +++++ workflow/rocoto/sfs_xml.py | 32 + workflow/rocoto/tasks.py | 382 ++- .../rocoto/tasks.py~develop_Feb212025_ba09498 | 381 +++ .../rocoto/tasks.py~gsl_ufs_dev_Mar05_af805b | 381 +++ workflow/rocoto/tasks_factory.py | 2 + workflow/rocoto/workflow_xml.py | 181 +- .../workflow_xml.py~develop_Feb212025_ba09498 | 180 ++ .../workflow_xml.py~gsl_ufs_dev_Mar05_af805b | 180 ++ workflow/setup_expt.py | 418 +-- workflow/tests/__init__.py | 0 workflow/tests/test_configuration.py | 32 + workflow/tests/test_hosts.py | 17 + workflow/tests/test_tasks.py | 84 + workflow/wxflow | 1 + 603 files changed, 30297 insertions(+), 7204 deletions(-) create mode 100644 .flake8 create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/dump_request.yml create mode 100644 .github/ISSUE_TEMPLATE/static_data.yml create mode 100644 .github/workflows/hercules.yaml create mode 100644 .github/workflows/pw_aws_ci.yaml create mode 100644 .github/workflows/wcoss2.yaml create mode 100644 CMakeLists.txt delete mode 100644 FV3GFSwfm/rt_c384ics/config.analcalc delete mode 100644 FV3GFSwfm/rt_c384ics/config.eobs delete mode 100644 FV3GFSwfm/rt_c384ics/config.epos delete mode 100644 FV3GFSwfm/rt_c384ics/config.esfc delete mode 100644 FV3GFSwfm/rt_c384ics/config.eupd delete mode 100644 FV3GFSwfm/rt_c384ics/config.wavepostsbs create mode 100644 ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml create mode 100644 ci/cases/gfsv17/marine3dvar.yaml create mode 100644 ci/cases/gfsv17/marinehyb.yaml create mode 100644 ci/cases/pr/C48_S2SW_extended.yaml create mode 100644 ci/cases/pr/C48mx500_hybAOWCDA.yaml create mode 100644 ci/cases/pr/C96C48_hybatmaerosnowDA.yaml create mode 100644 ci/cases/pr/C96mx100_S2S.yaml create mode 100644 ci/cases/sfs/C96mx100_S2S.yaml create mode 100644 ci/cases/yamls/gefs_defaults_ci.yaml create mode 100644 ci/cases/yamls/gefs_replay_ci.yaml create mode 100644 ci/cases/yamls/gfs_metp_ci.yaml create mode 100644 ci/cases/yamls/sfs_defaults.yaml create mode 100644 ci/cases/yamls/sfs_full.yaml create mode 100644 ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml create mode 100644 ci/platforms/config.gaeac5 create mode 100644 ci/platforms/config.gaeac6 create mode 100755 ci/scripts/utils/get_config_var.py create mode 100644 ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt create mode 100755 ci/scripts/utils/parallel_works/provision_runner.sh create mode 120000 ci/scripts/wxflow create mode 100644 ctests/CMakeLists.txt create mode 100644 ctests/README.md create mode 100644 ctests/cases/C48_ATM_gfs_fcst_seg0.yaml create mode 100644 ctests/cases/C48_S2SW_gfs_atmos_prod_f000-f003.yaml create mode 100644 ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml create mode 100755 ctests/scripts/execute.sh.in create mode 100755 ctests/scripts/setup.sh.in create mode 100755 ctests/scripts/stage.py create mode 100755 ctests/scripts/stage.sh.in create mode 100755 ctests/scripts/validate.py create mode 100755 ctests/scripts/validate.sh.in create mode 100644 docs/source/references.bib create mode 100755 env/AZUREPW.env create mode 100755 env/GAEAC5.env create mode 100755 env/GAEAC6.env create mode 100755 env/GOOGLEPW.env create mode 100755 jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX create mode 100755 jobs/JGDAS_ENKF_ARCHIVE_TARS create mode 100755 jobs/JGDAS_ENKF_ARCHIVE_VRFY create mode 100755 jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL create mode 100755 jobs/JGLOBAL_ARCHIVE_TARS create mode 100755 jobs/JGLOBAL_ARCHIVE_VRFY create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_OBS create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_SOL create mode 100755 jobs/JGLOBAL_FETCH create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL create mode 100755 jobs/JGLOBAL_SNOWENS_ANALYSIS create mode 100755 jobs/rocoto/aeroanlgenb.sh create mode 100755 jobs/rocoto/aeroanlvar.sh rename jobs/rocoto/{arch.sh => arch_tars.sh} (90%) create mode 100755 jobs/rocoto/arch_vrfy.sh create mode 100755 jobs/rocoto/atmensanlobs.sh create mode 100755 jobs/rocoto/atmensanlsol.sh create mode 100755 jobs/rocoto/atmos_products.sh.emc rename jobs/rocoto/{earc.sh => earc_tars.sh} (90%) create mode 100755 jobs/rocoto/earc_vrfy.sh create mode 100755 jobs/rocoto/esnowanl.sh create mode 100755 jobs/rocoto/fetch.sh create mode 100755 jobs/rocoto/marineanlchkpt.sh create mode 100755 jobs/rocoto/marineanlfinal.sh create mode 100755 jobs/rocoto/marineanlinit.sh create mode 100755 jobs/rocoto/marineanlletkf.sh create mode 100755 jobs/rocoto/marineanlvar.sh create mode 100644 modulefiles/module_base.gaeac5.lua create mode 100644 modulefiles/module_base.gaeac6.lua create mode 100644 modulefiles/module_base.noaacloud.lua create mode 100644 modulefiles/module_gwci.noaacloud.lua create mode 100644 modulefiles/module_gwsetup.gaeac5.lua create mode 100644 modulefiles/module_gwsetup.gaeac6.lua create mode 100644 modulefiles/module_gwsetup.noaacloud.lua create mode 100644 parm/archive/expdir.yaml.j2 create mode 100644 parm/archive/gefs_arcdir.yaml.j2 create mode 100644 parm/archive/gfs_arcdir.yaml.j2 create mode 100644 parm/archive/master_gefs.yaml.j2 create mode 100644 parm/config/gefs/config.arch_tars create mode 100644 parm/config/gefs/config.arch_vrfy create mode 120000 parm/config/gefs/config.cleanup create mode 120000 parm/config/gefs/config.fetch create mode 100644 parm/config/gefs/config.resources.AWSPW create mode 100644 parm/config/gefs/config.resources.AZUREPW create mode 100644 parm/config/gefs/config.resources.GOOGLEPW create mode 100644 parm/config/gfs/config.aeroanlgenb create mode 100644 parm/config/gfs/config.aeroanlvar create mode 100644 parm/config/gfs/config.arch_tars create mode 100644 parm/config/gfs/config.arch_vrfy create mode 100644 parm/config/gfs/config.atmensanlobs create mode 100644 parm/config/gfs/config.atmensanlsol mode change 120000 => 100644 parm/config/gfs/config.base create mode 100644 parm/config/gfs/config.base~gsl_ufs_dev_Mar05_af805b create mode 100644 parm/config/gfs/config.earc_tars rename FV3GFSwfm/rt_c384ics/config.earc => parm/config/gfs/config.earc_vrfy (52%) create mode 100644 parm/config/gfs/config.esnowanl create mode 100644 parm/config/gfs/config.fetch create mode 100644 parm/config/gfs/config.marineanl create mode 100644 parm/config/gfs/config.marineanlchkpt create mode 100644 parm/config/gfs/config.marineanlfinal create mode 100644 parm/config/gfs/config.marineanlinit create mode 100644 parm/config/gfs/config.marineanlletkf create mode 100644 parm/config/gfs/config.marineanlvar create mode 100644 parm/config/gfs/config.resources.AWSPW create mode 100644 parm/config/gfs/config.resources.AZUREPW create mode 100644 parm/config/gfs/config.resources.GAEAC5 create mode 100644 parm/config/gfs/config.resources.GAEAC6 create mode 100644 parm/config/gfs/config.resources.GOOGLEPW mode change 120000 => 100644 parm/config/gfs/config.ufs create mode 100644 parm/config/gfs/config.ufs~develop_Feb212025_ba09498 create mode 100644 parm/config/gfs/config.ufs~gsl_ufs_dev_Mar05_af805b create mode 120000 parm/config/sfs/config.aero create mode 120000 parm/config/sfs/config.arch_tars create mode 120000 parm/config/sfs/config.arch_vrfy create mode 120000 parm/config/sfs/config.atmos_ensstat create mode 120000 parm/config/sfs/config.atmos_products create mode 100644 parm/config/sfs/config.base create mode 120000 parm/config/sfs/config.cleanup create mode 120000 parm/config/sfs/config.com create mode 120000 parm/config/sfs/config.efcs create mode 120000 parm/config/sfs/config.extractvars create mode 120000 parm/config/sfs/config.fcst create mode 120000 parm/config/sfs/config.ice create mode 120000 parm/config/sfs/config.nsst create mode 120000 parm/config/sfs/config.oceanice_products create mode 120000 parm/config/sfs/config.ocn create mode 120000 parm/config/sfs/config.prep_emissions create mode 100644 parm/config/sfs/config.resources create mode 120000 parm/config/sfs/config.resources.AWSPW create mode 120000 parm/config/sfs/config.resources.AZUREPW create mode 120000 parm/config/sfs/config.resources.GOOGLEPW create mode 120000 parm/config/sfs/config.stage_ic create mode 120000 parm/config/sfs/config.ufs create mode 120000 parm/config/sfs/config.wave create mode 120000 parm/config/sfs/config.waveinit create mode 120000 parm/config/sfs/config.wavepostbndpnt create mode 120000 parm/config/sfs/config.wavepostbndpntbll create mode 120000 parm/config/sfs/config.wavepostpnt create mode 120000 parm/config/sfs/config.wavepostsbs create mode 100644 parm/config/sfs/yaml/defaults.yaml create mode 100755 parm/fetch/gfs_ATM_cold_forecast-only.yaml.j2 create mode 100755 parm/fetch/gfs_S2SW_cold_forecast-only.yaml.j2 create mode 100644 parm/gdas/aero_bmat_jedi_config.yaml.j2 create mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 create mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_stage_variational.yaml.j2 create mode 100644 parm/gdas/aeroanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/atmanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/atmensanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/esnowanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/snow_finalize_ens_update.yaml.j2 create mode 100644 parm/gdas/snow_stage_ens_update.yaml.j2 create mode 100644 parm/gdas/snow_stage_orog.yaml.j2 create mode 100644 parm/gdas/snowanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/soca_bmat_jedi_config.yaml.j2 create mode 100644 parm/gdas/staging/snow_berror.yaml.j2 create mode 100644 parm/gdas/staging/snow_var_bkg.yaml.j2 create mode 100644 parm/product/bufr_ij_gfs_C1152.txt rename parm/product/{bufr_ij13km.txt => bufr_ij_gfs_C768.txt} (94%) create mode 100644 parm/product/bufr_ij_gfs_C96.txt create mode 100644 parm/product/sfs.0p25.f000.paramlist.a.txt create mode 100644 parm/product/sfs.0p25.fFFF.paramlist.a.txt create mode 100644 parm/product/sfs.0p25.fFFF.paramlist.b.txt create mode 100644 parm/stage/analysis.yaml.j2 create mode 100644 parm/stage/atmosphere_cold.yaml.j2 create mode 100644 parm/stage/atmosphere_ens_perturbations.yaml.j2 create mode 100644 parm/stage/atmosphere_nest.yaml.j2 create mode 100644 parm/stage/atmosphere_warm.yaml.j2 create mode 100644 parm/stage/ice.yaml.j2 create mode 100644 parm/stage/master_gefs.yaml.j2 create mode 100644 parm/stage/master_gfs.yaml.j2 create mode 100644 parm/stage/ocean.yaml.j2 create mode 100644 parm/stage/ocean_ens_perturbations.yaml.j2 create mode 100644 parm/stage/ocean_mediator.yaml.j2 create mode 100644 parm/stage/ocean_rerun.yaml.j2 create mode 100644 parm/stage/wave.yaml.j2 delete mode 100644 parm/transfer/transfer_gdas_1c.list delete mode 100644 parm/transfer/transfer_gdas_enkf_enkf_05.list delete mode 100644 parm/transfer/transfer_gdas_enkf_enkf_10.list delete mode 100644 parm/transfer/transfer_gdas_enkf_enkf_15.list create mode 100644 parm/ufs/fv3/diag_table_replay create mode 100755 scripts/exgdas_aero_analysis_generate_bmatrix.py create mode 100755 scripts/exgdas_enkf_earc_tars.py create mode 100755 scripts/exgdas_enkf_earc_vrfy.py create mode 100755 scripts/exglobal_aero_analysis_variational.py create mode 100755 scripts/exglobal_archive_tars.py create mode 100755 scripts/exglobal_archive_vrfy.py create mode 100755 scripts/exglobal_atmens_analysis_obs.py create mode 100755 scripts/exglobal_atmens_analysis_sol.py create mode 100755 scripts/exglobal_fetch.py create mode 100755 scripts/exglobal_marine_analysis_checkpoint.py create mode 100755 scripts/exglobal_marine_analysis_finalize.py create mode 100755 scripts/exglobal_marine_analysis_initialize.py create mode 100755 scripts/exglobal_marine_analysis_letkf.py create mode 100755 scripts/exglobal_marine_analysis_variational.py create mode 100755 scripts/exglobal_snowens_analysis.py create mode 100755 scripts/exglobal_stage_ic.py create mode 100755 sorc/build_compute.sh delete mode 100644 sorc/ncl.setup delete mode 100755 ush/fv3gfs_remap.sh delete mode 120000 ush/hpssarch_gen.sh create mode 100755 ush/make_NTC_file.pl create mode 100755 ush/make_ntc_bull.pl create mode 100755 ush/make_tif.sh create mode 100755 ush/month_name.sh create mode 100644 ush/parsing_namelists_GOCART.sh create mode 100644 ush/python/pygfs/jedi/__init__.py create mode 100644 ush/python/pygfs/jedi/jedi.py create mode 100644 ush/python/pygfs/task/aero_bmatrix.py create mode 100755 ush/python/pygfs/task/fetch.py create mode 100644 ush/python/pygfs/task/marine_analysis.py create mode 100644 ush/python/pygfs/task/snowens_analysis.py create mode 100644 ush/python/pygfs/task/stage_ic.py create mode 120000 ush/python/wxflow create mode 100644 ush/wave_domain_grid.sh create mode 100644 versions/build.gaeac5.ver create mode 100644 versions/build.gaeac6.ver create mode 100644 versions/build.noaacloud.ver create mode 100644 versions/ic.ver create mode 100644 versions/run.gaeac5.ver create mode 100644 versions/run.gaeac6.ver create mode 100644 versions/run.noaacloud.ver create mode 100644 versions/spack.ver create mode 100644 workflow/applications/sfs.py create mode 100755 workflow/build_compute.py create mode 100644 workflow/build_opts.yaml create mode 100755 workflow/generate_workflows.sh create mode 100644 workflow/hosts/azurepw.yaml create mode 100644 workflow/hosts/gaeac5.yaml create mode 100644 workflow/hosts/gaeac6.yaml create mode 100644 workflow/hosts/googlepw.yaml mode change 120000 => 100644 workflow/hosts/hera.yaml create mode 100644 workflow/hosts/hera.yaml~develop_Feb212025_ba09498 create mode 100644 workflow/hosts/hera.yaml~gsl_ufs_dev_Mar05_af805b mode change 120000 => 100644 workflow/hosts/jet.yaml create mode 100644 workflow/hosts/jet.yaml~develop_Feb212025_ba09498 create mode 100644 workflow/hosts/jet.yaml~gsl_ufs_dev_Mar05_af805b create mode 100755 workflow/mynn.sh create mode 100644 workflow/rocoto/sfs_tasks.py create mode 100644 workflow/rocoto/sfs_xml.py mode change 120000 => 100644 workflow/rocoto/tasks.py create mode 100644 workflow/rocoto/tasks.py~develop_Feb212025_ba09498 create mode 100644 workflow/rocoto/tasks.py~gsl_ufs_dev_Mar05_af805b mode change 120000 => 100644 workflow/rocoto/workflow_xml.py create mode 100644 workflow/rocoto/workflow_xml.py~develop_Feb212025_ba09498 create mode 100644 workflow/rocoto/workflow_xml.py~gsl_ufs_dev_Mar05_af805b create mode 100644 workflow/tests/__init__.py create mode 100644 workflow/tests/test_configuration.py create mode 100644 workflow/tests/test_hosts.py create mode 100644 workflow/tests/test_tasks.py create mode 120000 workflow/wxflow diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..e3a4e3e47ea --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +exclude = .git,.github,venv,__pycache__,old,build,dist +max-line-length = 160 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..6a9cf9d47c6 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,214 @@ +# global-workflow is a collaborative space where contributions come from a variety of sources +# This file is to ensure that new development gets reviewed by the appropriate SME + +# global-workflow default owners (not a complete list) +@KateFriedman-NOAA +@WalterKolczynski-NOAA +@DavidHuber-NOAA + +# Specific directory owners +/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA + +/ecf/ @lgannoaa + +/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA + +# Specific file owners +# build scripts +sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA +sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA + +# jobs +jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @CoryMartin-NOAA +jobs/JGDAS_ATMOS_ANALYSIS_DIAG @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_CHGRES_FORENKF @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGDAS_ATMOS_GEMPAK_META_NCDC @GwenChen-NOAA +jobs/JGDAS_ATMOS_VERFOZN @EdwardSafford-NOAA +jobs/JGDAS_ATMOS_VERFRAD @EdwardSafford-NOAA +jobs/JGDAS_ENKF_* @RussTreadon-NOAA @CoryMartin-NOAA @CatherineThomas-NOAA +jobs/JGDAS_FIT2OBS @jack-woollen +jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @guillaumevernieres +jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG @GwenChen-NOAA +jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA +jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA +jobs/JGFS_ATMOS_FBWIND @GwenChen-NOAA +jobs/JGFS_ATMOS_FSU_GENESIS +jobs/JGFS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_META @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC @GwenChen-NOAA +jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS @WenMeng-NOAA +jobs/JGFS_ATMOS_POSTSND @BoCui-NOAA +jobs/JGFS_ATMOS_VERIFICATION +jobs/JGLOBAL_AERO_ANALYSIS_* @CoryMartin-NOAA +jobs/JGLOBAL_ARCHIVE @DavidHuber-NOAA +jobs/JGLOBAL_ATMENS_ANALYSIS_* @RussTreadon-NOAA @CoryMartin-NOAA @DavidNew-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS @RussTreadon-NOAA @CatherineThomas-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @RussTreadon-NOAA @CatherineThomas-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_ENSSTAT +jobs/JGLOBAL_ATMOS_POST_MANAGER +jobs/JGLOBAL_ATMOS_PRODUCTS @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_SFCANL @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC +jobs/JGLOBAL_ATMOS_UPP @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_VMINMON @EdwardSafford-NOAA +jobs/JGLOBAL_ATM_* @RussTreadon-NOAA @DavidNew-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_CLEANUP @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA +jobs/JGLOBAL_EXTRACTVARS @EricSinsky-NOAA +jobs/JGLOBAL_FORECAST @aerorahul +jobs/JGLOBAL_MARINE_* @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_OCEANICE_PRODUCTS @GwenChen-NOAA +jobs/JGLOBAL_PREP_EMISSIONS @bbakernoaa +jobs/JGLOBAL_PREP_OBS_AERO @CoryMartin-NOAA +jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_*SNOW* @jiaruidong2017 +jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA +jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash +jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA + +# scripts +scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA +scripts/exgdas_atmos_chgres_forenkf.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA +scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA +scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA +scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA +scripts/exgdas_enkf_earc_vrfy.py @DavidHuber-NOAA +scripts/exgdas_enkf_earc_tars.py @DavidHuber-NOAA +scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_sfc.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_snow_recenter.py @jiaruidong2017 +scripts/exgdas_enkf_update.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_global_marine_analysis_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exgfs_aero_init_aerosol.py @WalterKolczynski-NOAA +scripts/exgfs_atmos_awips_20km_1p0deg.sh @GwenChen-NOAA +scripts/exgfs_atmos_fbwind.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_gif_ncdc_skew_t.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_meta.sh @GwenChen-NOAA +scripts/exgfs_atmos_goes_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_grib2_special_npoess.sh @WenMeng-NOAA +scripts/exgfs_atmos_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_postsnd.sh @BoCui-NOAA +scripts/exgfs_pmgr.sh +scripts/exgfs_prdgen_manager.sh +scripts/exgfs_wave_* @JessicaMeixner-NOAA @sbanihash +scripts/exglobal_aero_analysis_* @CoryMartin-NOAA +scripts/exglobal_archive.py @DavidHuber-NOAA +scripts/exglobal_atm_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmens_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmos_analysis*.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_atmos_ensstat.sh @RussTreadon-NOAA +scripts/exglobal_atmos_pmgr.sh +scripts/exglobal_atmos_products.sh @WenMeng-NOAA +scripts/exglobal_atmos_sfcanl.sh @GeorgeGayno-NOAA +scripts/exglobal_atmos_tropcy_qc_reloc.sh +scripts/exglobal_atmos_upp.py @WenMeng-NOAA +scripts/exglobal_atmos_vminmon.sh @EdwardSafford-NOAA +scripts/exglobal_cleanup.sh @DavidHuber-NOAA +scripts/exglobal_diag.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_extractvars.sh @EricSinsky-NOAA +scripts/exglobal_forecast.py @aerorahul +scripts/exglobal_forecast.sh @aerorahul @WalterKolczynski-NOAA +scripts/exglobal_marine_analysis_* @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_marinebmat.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_oceanice_products.py @GwenChen-NOAA +scripts/exglobal_prep_emissions.py @bbakernoaa +scripts/exglobal_prep_obs_aero.py @CoryMartin-NOAA +scripts/exglobal_prep_snow_obs.py @jiaruidong2017 +scripts/exglobal_snow_analysis.py @jiaruidong2017 +scripts/exglobal_stage_ic.py @KateFriedman-NOAA + +# ush +WAM_XML_to_ASCII.pl +atmos_ensstat.sh +atmos_extractvars.sh @EricSinsky-NOAA +bash_utils.sh @WalterKolczynski-NOAA +calcanl_gfs.py @CoryMartin-NOAA +calcinc_gfs.py @CoryMartin-NOAA +compare_f90nml.py @WalterKolczynski-NOAA @aerorahul +detect_machine.sh @WalterKolczynski-NOAA +extractvars_tools.sh @EricSinsky-NOAA +file_utils.sh @WalterKolczynski-NOAA +forecast_det.sh @aerorahul @WalterKolczynski-NOAA +forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA +forecast_predet.sh @aerorahul @WalterKolczynski-NOAA +fv3gfs_remap_weights.sh +gaussian_sfcanl.sh @GeorgeGayno-NOAA +getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getgfsnctime @CoryMartin-NOAA +getncdimlen @CoryMartin-NOAA +gfs_bfr2gpk.sh @GwenChen-NOAA +gfs_bufr.sh @GwenChen-NOAA +gfs_bufr_netcdf.sh @GwenChen-NOAA +gfs_sndp.sh @BoCui-NOAA +gfs_truncate_enkf.sh @CoryMartin-NOAA +global_savefits.sh +gsi_utils.py @CoryMartin-NOAA +interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +jjob_header.sh @WalterKolczynski-NOAA +link_crtm_fix.sh @WalterKolczynski-NOAA +load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul +load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA +load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA +merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA +minmon_xtrct_*.pl @EdwardSafford-NOAA +module-setup.sh @WalterKolczynski-NOAA @aerorahul +oceanice_nc2grib2.sh @GwenChen-NOAA +ocnice_extractvars.sh @EricSinsky-NOAA +ozn_xtrct.sh @EdwardSafford-NOAA +parse-storm-type.pl +parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen +parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_FV3_nest.sh @guoqing-noaa +parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang +parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash +parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +preamble.sh @WalterKolczynski-NOAA +product_functions.sh @WalterKolczynski-NOAA @aerorahul +radmon_*.sh @EdwardSafford-NOAA +rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA +run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA +syndat_getjtbul.sh @JiayiPeng-NOAA +syndat_qctropcy.sh @JiayiPeng-NOAA +tropcy_relocate.sh @JiayiPeng-NOAA +tropcy_relocate_extrkr.sh @JiayiPeng-NOAA +wave_*.sh @JessicaMeixner-NOAA @sbanihash + +# ush/python +ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA +ush/python/pygfs/jedi/jedi.py @DavidNew-NOAA +ush/python/pygfs/task/__init__.py @aerorahul +ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_emissions.py @bbakernoaa +ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA +ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/archive.py @DavidHuber-NOAA +ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/bmatrix.py @DavidNew-NOAA +ush/python/pygfs/task/gfs_forecast.py @aerorahul +ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/oceanice_products.py @aerorahul @GwenChen-NOAA +ush/python/pygfs/task/snow_analysis.py @jiaruidong2017 +ush/python/pygfs/task/snowens_analysis.py @jiaruidong2017 +ush/python/pygfs/task/stage_ic.py @KateFriedman-NOAA +ush/python/pygfs/task/upp.py @aerorahul @WenMeng-NOAA +ush/python/pygfs/ufswm/__init__.py @aerorahul +ush/python/pygfs/ufswm/gfs.py @aerorahul +ush/python/pygfs/ufswm/ufs.py @aerorahul +ush/python/pygfs/utils/__init__.py @aerorahul +ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA + +# Specific workflow scripts +workflow/generate_workflows.sh @DavidHuber-NOAA +workflow/build_compute.py @DavidHuber-NOAA @aerorahul diff --git a/.github/ISSUE_TEMPLATE/NCO_bug_report.yml b/.github/ISSUE_TEMPLATE/NCO_bug_report.yml index cc532058077..79632779aa4 100644 --- a/.github/ISSUE_TEMPLATE/NCO_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/NCO_bug_report.yml @@ -1,4 +1,4 @@ -name: NCO Bug report +name: NCO Bug Report description: Report something that is incorrect or broken labels: ["nco-bug", "triage"] assignees: diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 216293781c5..d43d2eb3d72 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,4 +1,4 @@ -name: Bug report +name: Bug Report description: Report something that is incorrect or broken labels: ["bug", "triage"] @@ -9,6 +9,8 @@ body: Your bug may already be reported! Please search on the [Issue tracker](https://github.com/NOAA-EMC/global-workflow/issues) before creating one. + Is this actually a workflow bug? If not, please open an issue in the appropriate repository first. + - type: textarea id: current_behavior attributes: @@ -46,6 +48,13 @@ body: validations: required: true + - type: input + id: hash + attributes: + label: What global-workflow hash are you using? + validations: + required: true + - type: textarea id: reproduction_steps attributes: @@ -63,8 +72,9 @@ body: attributes: label: Additional information description: Provide context or any additional information about the bug. + placeholder: Optional validations: - required: true + required: false - type: textarea id: propsed_implementation diff --git a/.github/ISSUE_TEMPLATE/dump_request.yml b/.github/ISSUE_TEMPLATE/dump_request.yml new file mode 100644 index 00000000000..4481e7f6e05 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dump_request.yml @@ -0,0 +1,37 @@ +name: Global Observation Dump Request +description: Request additional dates be added to a machine's global dump archive (GDA) or introduce experimental dump data to the GDA +labels: ["Static Data Mgmt"] +assignees: + - KateFriedman-NOAA + - WalterKolczynski-NOAA + +body: + - type: dropdown + attributes: + label: Machine + options: + - WCOSS2 + - Hera/Ursa + - Orion/Hercules + - Jet + multiple: true + validations: + required: true + + - type: input + attributes: + label: Start date + validations: + required: true + + - type: input + attributes: + label: End date + validations: + required: true + + - type: textarea + attributes: + label: Additional information + placeholder: | + Any additional information needed (experimental obs, etc.) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 3aed58c520c..7e0ddb24590 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,4 +1,4 @@ -name: Feature request +name: Feature Request description: Request new capability labels: ["feature", "triage"] @@ -10,8 +10,7 @@ body: - Something is wrong or broken - It is a request from NCO - It is initiating a production update - - Files need to be updated or added to fix - + - Files need to be updated in a directory managed by workflow CMs under "glopara" Please search on the [Issue tracker](https://github.com/NOAA-EMC/global-workflow/issues) to make sure the feature has not already been requested to avoid duplicates. @@ -34,7 +33,17 @@ body: id: criteria attributes: label: Acceptance Criteria - placeholder: What does it mean for this issue to be complete? + description: | + A checklist of criteria to be satisfied before this feature is considered complete. + + Examples: + - Forecast with option X completes successfully + - File X produced + - Output unchanged + placeholder: | + - [ ] Criterion #1 + - [ ] Criterion #2 + - [ ] Criterion #3 validations: required: true diff --git a/.github/ISSUE_TEMPLATE/production_update.yml b/.github/ISSUE_TEMPLATE/production_update.yml index ac9ada505a5..cb1fb588d85 100644 --- a/.github/ISSUE_TEMPLATE/production_update.yml +++ b/.github/ISSUE_TEMPLATE/production_update.yml @@ -33,18 +33,19 @@ body: validations: required: true - - type: markdown + - type: checkboxes attributes: - value: | - ** Tasks ** - - [ ] Create release branch - - [ ] Make workflow changes for upgrade in release branch (add additional checklist items as needed) - - [ ] Create release notes - - [ ] Cut hand-off tag for CDF - - [ ] Submit CDF to NCO - - [ ] Implementation into operations complete - - [ ] Merge release branch into operational branch - - [ ] Cut version tag from operational branch - - [ ] Release new version tag - - [ ] Announce to users - - [ ] Update Read-The-Docs operations status version in develop + label: Tasks + description: List of tasks to complete update. For use after issue is created. + options: + - label: Create release branch + - label: Make workflow changes for upgrade in release branch (add additional checklist items as needed) + - label: Create release notes + - label: Cut hand-off tag for CDF + - label: Submit CDF to NCO + - label: Implementation into operations complete + - label: Merge release branch into operational branch + - label: Cut version tag from operational branch + - label: Release new version tag + - label: Announce to users + - label: Update Read-The-Docs operations status version in develop diff --git a/.github/ISSUE_TEMPLATE/static_data.yml b/.github/ISSUE_TEMPLATE/static_data.yml new file mode 100644 index 00000000000..f29f155cf8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/static_data.yml @@ -0,0 +1,134 @@ +name: Static Data Update +description: Request static data be added or updated +labels: ["Static Data Mgmt"] +assignees: + - KateFriedman-NOAA + - WalterKolczynski-NOAA + +body: + - type: dropdown + attributes: + label: Type of static data + description: | + - Fix: large static files needed to run global-workflow + - Initial conditions: ICs needed to run one of global-workflow's included tests + - Experimental data: Temporary datasets needed for development that will be replaced by other sources for operations (e.g. data that will be in DCOM) + options: + - Fix + - Initial conditions + - Experimental data + validations: + required: true + + - type: dropdown + attributes: + label: Target directory + options: + - "Fix: aer" + - "Fix: am" + - "Fix: chem" + - "Fix: cice" + - "Fix: cpl" + - "Fix: datm" + - "Fix: gdas/crtm" + - "Fix: gdas/fv3jedi" + - "Fix: gdas/gsibec" + - "Fix: gldas" + - "Fix: glwu" + - "Fix: gsi" + - "Fix: lut" + - "Fix: mom6" + - "Fix: orog" + - "Fix: raw" + - "Fix: reg2grb2" + - "Fix: sfc_climo" + - "Fix: ugwd" + - "Fix: verif" + - "Fix: wave" + - "Fix: *NEW*" + - "ICs: C48/C48 mx500" + - "ICs: C96/C48" + - "ICs: C96 mx100" + - "ICs: C384/C192" + - "ICs: C384 mx025" + - "ICs: C768 mx025" + - "ICs: C1152 mx025" + - "ICs: *NEW RESOLUTION*" + - "Exp Data: GOCART emissions" + - "Exp Data: JEDI obs" + - "Exp Data: *NEW*" + multiple: true + validations: + required: true + + - type: checkboxes + attributes: + label: Type of change + options: + - label: Update existing files (Version update needed) + - label: Remove files (Version update needed) + - label: Add new files to existing directory + - label: Add new directory (Version update needed if fix) + validations: + required: true + + - type: dropdown + attributes: + label: Any workflow changes needed? + description: | + ANY fix or IC change other than adding files will require at least a new version. + options: + - No change needed + - Fix/IC version update only + - Fix/IC version update + additional changes + - Non-version changes only + validations: + required: true + + - type: textarea + attributes: + label: | + Please list all related issues. If this request requires a workflow update (including a fix version update), please open a *separate* global-workflow issue to track the change. + description: Please link any related issues + placeholder: NOAA-EMC/repo#1234 + validations: + required: false + + - type: input + attributes: + label: Pickup location + description: | + Please provide a path to the data on either Hera or WCOSS. + + If this requires a version update, please provide a CLEAN copy of the *entire* directory, including the structure and unchanged files from current directory. + + If this is just adding additional files, you may provide just the new files. The directory structure should still match the target. + placeholder: '/path/to/new/data' + validations: + required: true + + - type: input + attributes: + label: Size + description: How much data needs to be copied (size of pickup location in MB)? + placeholder: '0' + validations: + required: true + + - type: textarea + attributes: + label: Additional information + description: | + Please provide any additional information needed for this request. If this is a new directory, please provide a short description and a point of contact. + validations: + required: false + + - type: markdown + attributes: + value: | + # Tasks + - [ ] Discuss needs with global-workflow developer assigned to request + - [ ] Stage a CLEAN copy of data on Hera or WCOSS in the location provided above + - [ ] [global-workflow CM] Update data in "glopara"-managed space on supported platforms + - [ ] [Fix only] Update "Fix File Management" [spreadsheet](https://docs.google.com/spreadsheets/d/1BeIvcz6TO3If4YCqkUK-oz_kGS9q2wTjwLS-BBemSEY/edit?usp=sharing) + - [ ] Make related workflow/component updates (if any) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3f8fe650652..9e9c9eccfec 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,19 +1,28 @@ - - # Description - - + + # Type of change - -- Bug fix (fixes something broken) -- New feature (adds functionality) -- Maintenance (code refactor, clean-up, new CI test, etc.) +- [ ] Bug fix (fixes something broken) +- [ ] New feature (adds functionality) +- [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics + - Is this a breaking change (a change in existing functionality)? YES/NO - Does this change require a documentation update? YES/NO - Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.) - - [ ] EMC verif-global - - [ ] GDAS - - [ ] GFS-utils - - [ ] GSI - - [ ] GSI-monitor - - [ ] GSI-utils - - [ ] UFS-utils - - [ ] UFS-weather-model - - [ ] wxflow - + - [ ] EMC verif-global + - [ ] GDAS + - [ ] GFS-utils + - [ ] GSI + - [ ] GSI-monitor + - [ ] GSI-utils + - [ ] UFS-utils + - [ ] UFS-weather-model + - [ ] wxflow # How has this been tested? ', + ''] + + return '\n'.join(strings) + + def _get_definitions(self) -> str: + """ + Create entities related to the experiment + """ + + entity = OrderedDict() + + entity['PSLOT'] = self._base['PSLOT'] + + entity['ROTDIR'] = self._base['ROTDIR'] + entity['JOBS_DIR'] = self._base['BASE_JOB'] + + entity['MAXTRIES'] = self.rocoto_config['maxtries'] + + # Put them all in an XML key-value syntax + strings = [] + for key, value in entity.items(): + strings.append('\t' + rocoto.create_entity(key, value)) + + return '\n'.join(strings) + + def _get_workflow_header(self): + """ + Create the workflow header block + """ + + scheduler = self._app_config.scheduler + cyclethrottle = self.rocoto_config['cyclethrottle'] + taskthrottle = self.rocoto_config['taskthrottle'] + verbosity = self.rocoto_config['verbosity'] + + expdir = self._base['EXPDIR'] + + strings = ['', + ']>', + '', + f'', + '', + f'\t{expdir}/logs/@Y@m@d@H.log', + '', + '\t', + ''] + + return '\n'.join(strings) + + @abstractmethod + def get_cycledefs(self): + pass + + @staticmethod + def _get_workflow_footer(): + """ + Generate workflow footer + """ + + return '\n\n' + + def _assemble_xml(self) -> str: + + strings = [self.preamble, + self.definitions, + self.header, + self.cycledefs, + self.tasks, + self.footer] + + return ''.join(strings) + + def write(self, xml_file: str = None, crontab_file: str = None): + self._write_xml(xml_file=xml_file) + self._write_crontab(crontab_file=crontab_file) + + def _write_xml(self, xml_file: str = None) -> None: + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + if xml_file is None: + xml_file = f"{expdir}/{pslot}.xml" + + with open(xml_file, 'w') as fh: + fh.write(self.xml) + + def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: + """ + Create crontab to execute rocotorun every cronint (5) minutes + """ + + # No point creating a crontab if rocotorun is not available. + rocotoruncmd = find_executable('rocotorun') + if rocotoruncmd is None: + print('Failed to find rocotorun, crontab will not be created') + return + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + rocotorunstr = f'{rocotoruncmd} -d {expdir}/{pslot}.db -w {expdir}/{pslot}.xml' + cronintstr = f'*/{cronint} * * * *' + + try: + replyto = os.environ['REPLYTO'] + except KeyError: + replyto = '' + + strings = ['', + f'#################### {pslot} ####################', + f'MAILTO="{replyto}"' + ] + # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start. + if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']: + strings.extend([f'SHELL="/bin/bash"', + f'BASH_ENV="/etc/bashrc"']) + strings.extend([f'{cronintstr} {rocotorunstr}', + '#################################################################', + '']) + + if crontab_file is None: + crontab_file = f"{expdir}/{pslot}.crontab" + + with open(crontab_file, 'w') as fh: + fh.write('\n'.join(strings)) + + return diff --git a/workflow/rocoto/workflow_xml.py~develop_Feb212025_ba09498 b/workflow/rocoto/workflow_xml.py~develop_Feb212025_ba09498 new file mode 100644 index 00000000000..bed19ad5eea --- /dev/null +++ b/workflow/rocoto/workflow_xml.py~develop_Feb212025_ba09498 @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +import os +from distutils.spawn import find_executable +from datetime import datetime +from collections import OrderedDict +from typing import Dict +from applications.applications import AppConfig +from rocoto.workflow_tasks import get_wf_tasks +from wxflow import to_timedelta +import rocoto.rocoto as rocoto +from abc import ABC, abstractmethod + + +class RocotoXML(ABC): + + def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: + + self._app_config = app_config + self.rocoto_config = rocoto_config + + # Use the first config.base (sourced with an arbitrary RUN) + self._base = self._app_config.configs[next(iter(self._app_config.configs))]['base'] + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') + + self.preamble = self._get_preamble() + self.definitions = self._get_definitions() + self.header = self._get_workflow_header() + self.cycledefs = self.get_cycledefs() + task_list = get_wf_tasks(app_config) + self.tasks = '\n'.join(task_list) + self.footer = self._get_workflow_footer() + + self.xml = self._assemble_xml() + + @staticmethod + def _get_preamble(): + """ + Generate preamble for XML + """ + + strings = ['', + '', + ''] + + return '\n'.join(strings) + + def _get_definitions(self) -> str: + """ + Create entities related to the experiment + """ + + entity = OrderedDict() + + entity['PSLOT'] = self._base['PSLOT'] + + entity['ROTDIR'] = self._base['ROTDIR'] + entity['JOBS_DIR'] = self._base['BASE_JOB'] + + entity['MAXTRIES'] = self.rocoto_config['maxtries'] + + # Put them all in an XML key-value syntax + strings = [] + for key, value in entity.items(): + strings.append('\t' + rocoto.create_entity(key, value)) + + return '\n'.join(strings) + + def _get_workflow_header(self): + """ + Create the workflow header block + """ + + scheduler = self._app_config.scheduler + cyclethrottle = self.rocoto_config['cyclethrottle'] + taskthrottle = self.rocoto_config['taskthrottle'] + verbosity = self.rocoto_config['verbosity'] + + expdir = self._base['EXPDIR'] + + strings = ['', + ']>', + '', + f'', + '', + f'\t{expdir}/logs/@Y@m@d@H.log', + '', + '\t', + ''] + + return '\n'.join(strings) + + @abstractmethod + def get_cycledefs(self): + pass + + @staticmethod + def _get_workflow_footer(): + """ + Generate workflow footer + """ + + return '\n\n' + + def _assemble_xml(self) -> str: + + strings = [self.preamble, + self.definitions, + self.header, + self.cycledefs, + self.tasks, + self.footer] + + return ''.join(strings) + + def write(self, xml_file: str = None, crontab_file: str = None): + self._write_xml(xml_file=xml_file) + self._write_crontab(crontab_file=crontab_file) + + def _write_xml(self, xml_file: str = None) -> None: + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + if xml_file is None: + xml_file = f"{expdir}/{pslot}.xml" + + with open(xml_file, 'w') as fh: + fh.write(self.xml) + + def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: + """ + Create crontab to execute rocotorun every cronint (5) minutes + """ + + # No point creating a crontab if rocotorun is not available. + rocotoruncmd = find_executable('rocotorun') + if rocotoruncmd is None: + print('Failed to find rocotorun, crontab will not be created') + return + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + rocotorunstr = f'{rocotoruncmd} -d {expdir}/{pslot}.db -w {expdir}/{pslot}.xml' + cronintstr = f'*/{cronint} * * * *' + + try: + replyto = os.environ['REPLYTO'] + except KeyError: + replyto = '' + + strings = ['', + f'#################### {pslot} ####################', + f'MAILTO="{replyto}"' + ] + # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start. + if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']: + strings.extend([f'SHELL="/bin/bash"', + f'BASH_ENV="/etc/bashrc"']) + strings.extend([f'{cronintstr} {rocotorunstr}', + '#################################################################', + '']) + + if crontab_file is None: + crontab_file = f"{expdir}/{pslot}.crontab" + + with open(crontab_file, 'w') as fh: + fh.write('\n'.join(strings)) + + return diff --git a/workflow/rocoto/workflow_xml.py~gsl_ufs_dev_Mar05_af805b b/workflow/rocoto/workflow_xml.py~gsl_ufs_dev_Mar05_af805b new file mode 100644 index 00000000000..bed19ad5eea --- /dev/null +++ b/workflow/rocoto/workflow_xml.py~gsl_ufs_dev_Mar05_af805b @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +import os +from distutils.spawn import find_executable +from datetime import datetime +from collections import OrderedDict +from typing import Dict +from applications.applications import AppConfig +from rocoto.workflow_tasks import get_wf_tasks +from wxflow import to_timedelta +import rocoto.rocoto as rocoto +from abc import ABC, abstractmethod + + +class RocotoXML(ABC): + + def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: + + self._app_config = app_config + self.rocoto_config = rocoto_config + + # Use the first config.base (sourced with an arbitrary RUN) + self._base = self._app_config.configs[next(iter(self._app_config.configs))]['base'] + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') + + self.preamble = self._get_preamble() + self.definitions = self._get_definitions() + self.header = self._get_workflow_header() + self.cycledefs = self.get_cycledefs() + task_list = get_wf_tasks(app_config) + self.tasks = '\n'.join(task_list) + self.footer = self._get_workflow_footer() + + self.xml = self._assemble_xml() + + @staticmethod + def _get_preamble(): + """ + Generate preamble for XML + """ + + strings = ['', + '', + ''] + + return '\n'.join(strings) + + def _get_definitions(self) -> str: + """ + Create entities related to the experiment + """ + + entity = OrderedDict() + + entity['PSLOT'] = self._base['PSLOT'] + + entity['ROTDIR'] = self._base['ROTDIR'] + entity['JOBS_DIR'] = self._base['BASE_JOB'] + + entity['MAXTRIES'] = self.rocoto_config['maxtries'] + + # Put them all in an XML key-value syntax + strings = [] + for key, value in entity.items(): + strings.append('\t' + rocoto.create_entity(key, value)) + + return '\n'.join(strings) + + def _get_workflow_header(self): + """ + Create the workflow header block + """ + + scheduler = self._app_config.scheduler + cyclethrottle = self.rocoto_config['cyclethrottle'] + taskthrottle = self.rocoto_config['taskthrottle'] + verbosity = self.rocoto_config['verbosity'] + + expdir = self._base['EXPDIR'] + + strings = ['', + ']>', + '', + f'', + '', + f'\t{expdir}/logs/@Y@m@d@H.log', + '', + '\t', + ''] + + return '\n'.join(strings) + + @abstractmethod + def get_cycledefs(self): + pass + + @staticmethod + def _get_workflow_footer(): + """ + Generate workflow footer + """ + + return '\n\n' + + def _assemble_xml(self) -> str: + + strings = [self.preamble, + self.definitions, + self.header, + self.cycledefs, + self.tasks, + self.footer] + + return ''.join(strings) + + def write(self, xml_file: str = None, crontab_file: str = None): + self._write_xml(xml_file=xml_file) + self._write_crontab(crontab_file=crontab_file) + + def _write_xml(self, xml_file: str = None) -> None: + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + if xml_file is None: + xml_file = f"{expdir}/{pslot}.xml" + + with open(xml_file, 'w') as fh: + fh.write(self.xml) + + def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: + """ + Create crontab to execute rocotorun every cronint (5) minutes + """ + + # No point creating a crontab if rocotorun is not available. + rocotoruncmd = find_executable('rocotorun') + if rocotoruncmd is None: + print('Failed to find rocotorun, crontab will not be created') + return + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + rocotorunstr = f'{rocotoruncmd} -d {expdir}/{pslot}.db -w {expdir}/{pslot}.xml' + cronintstr = f'*/{cronint} * * * *' + + try: + replyto = os.environ['REPLYTO'] + except KeyError: + replyto = '' + + strings = ['', + f'#################### {pslot} ####################', + f'MAILTO="{replyto}"' + ] + # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start. + if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']: + strings.extend([f'SHELL="/bin/bash"', + f'BASH_ENV="/etc/bashrc"']) + strings.extend([f'{cronintstr} {rocotorunstr}', + '#################################################################', + '']) + + if crontab_file is None: + crontab_file = f"{expdir}/{pslot}.crontab" + + with open(crontab_file, 'w') as fh: + fh.write('\n'.join(strings)) + + return diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 3e70df0f021..88958b13b67 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,8 +7,7 @@ import os import glob import shutil -import warnings -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host @@ -29,236 +28,7 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_ROTDIR(host, inputs): - """ - Method to populate the ROTDIR for supported modes. - INPUTS: - host: host object from class Host - inputs: user inputs to setup_expt.py - """ - - fill_modes = { - 'cycled': fill_ROTDIR_cycled, - 'forecast-only': fill_ROTDIR_forecasts - } - - try: - fill_modes[inputs.mode](host, inputs) - except KeyError: - raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + - 'Currently supported modes are:\n' + - f'{" | ".join(fill_modes.keys())}') - - return - - -def fill_ROTDIR_cycled(host, inputs): - """ - Implementation of 'fill_ROTDIR' for cycled mode - """ - - rotdir = os.path.join(inputs.comroot, inputs.pslot) - - do_ocean = do_ice = do_med = False - - if 'S2S' in inputs.app: - do_ocean = do_ice = do_med = True - - if inputs.icsdir is None: - warnings.warn("User did not provide '--icsdir' to stage initial conditions") - return - - rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) - idatestr = datetime_to_YMDH(inputs.idate) - - # Test if we are using the new COM structure or the old flat one for ICs - if inputs.start in ['warm']: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') - else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') - - if os.path.isdir(pathstr): - flat_structure = False - else: - flat_structure = True - - # Destination always uses the new COM structure - # These should match the templates defined in config.com - if inputs.start in ['warm']: - dst_atm_dir = os.path.join('model_data', 'atmos', 'restart') - dst_med_dir = os.path.join('model_data', 'med', 'restart') - else: - dst_atm_dir = os.path.join('model_data', 'atmos', 'input') - dst_med_dir = '' # no mediator files for a "cold start" - do_med = False - dst_ocn_rst_dir = os.path.join('model_data', 'ocean', 'restart') - dst_ocn_anl_dir = os.path.join('analysis', 'ocean') - dst_ice_rst_dir = os.path.join('model_data', 'ice', 'restart') - dst_ice_anl_dir = os.path.join('analysis', 'ice') - dst_atm_anl_dir = os.path.join('analysis', 'atmos') - - if flat_structure: - # ICs are in the old flat COM structure - if inputs.start in ['warm']: # This is warm start experiment - src_atm_dir = os.path.join('atmos', 'RESTART') - src_med_dir = os.path.join('med', 'RESTART') - elif inputs.start in ['cold']: # This is a cold start experiment - src_atm_dir = os.path.join('atmos', 'INPUT') - src_med_dir = '' # no mediator files for a "cold start" - do_med = False - # ocean and ice have the same filenames for warm and cold - src_ocn_rst_dir = os.path.join('ocean', 'RESTART') - src_ocn_anl_dir = 'ocean' - src_ice_rst_dir = os.path.join('ice', 'RESTART') - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = 'atmos' - else: - src_atm_dir = dst_atm_dir - src_med_dir = dst_med_dir - src_ocn_rst_dir = dst_ocn_rst_dir - src_ocn_anl_dir = dst_ocn_anl_dir - src_ice_rst_dir = dst_ice_rst_dir - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = dst_atm_anl_dir - - def link_files_from_src_to_dst(src_dir, dst_dir): - files = os.listdir(src_dir) - for fname in files: - os.symlink(os.path.join(src_dir, fname), - os.path.join(dst_dir, fname)) - return - - # Link ensemble member initial conditions - if inputs.nens > 0: - previous_cycle_dir = f'enkf{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'enkf{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link deterministic initial conditions - previous_cycle_dir = f'{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_dir) - - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - # First 1/2 cycle needs a CICE6 analysis restart - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_ice_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, src_ice_anl_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link bias correction and radiance diagnostics files - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - # First 1/2 cycle also needs a atmos increment if doing warm start - if inputs.start in ['warm']: - for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - if inputs.nens > 0: - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc']: - fname = f'enkf{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - - return - - -def fill_ROTDIR_forecasts(host, inputs): - """ - Implementation of 'fill_ROTDIR' for forecast-only mode - """ - print('forecast-only mode treats ICs differently and cannot be staged here') - - -def fill_EXPDIR(inputs): +def fill_expdir(inputs): """ Method to copy config files from workflow to experiment directory INPUTS: @@ -279,52 +49,74 @@ def fill_EXPDIR(inputs): def update_configs(host, inputs): def _update_defaults(dict_in: dict) -> dict: + # Given an input dict_in of the form + # {defaults: {config_name: {var1: value1, ...}, }, config_name: {var1: value1, ...}} + # Replace values in ['defaults']['config_name']['var1'] with ['config_name']['var1'] + # and return the ['defaults'] subdictionary as its own new dictionary. defaults = dict_in.pop('defaults', AttrDict()) + if 'defaults' in defaults: + _update_defaults(defaults) defaults.update(dict_in) return defaults - # Read in the YAML file to fill out templates and override host defaults + # Convert the inputs to an AttrDict data = AttrDict(host.info, **inputs.__dict__) + + # Read in the YAML file to fill out templates data.HOMEgfs = _top yaml_path = inputs.yaml if not os.path.exists(yaml_path): - raise IOError(f'YAML file does not exist, check path:' + yaml_path) - yaml_dict = _update_defaults(AttrDict(parse_j2yaml(yaml_path, data))) + raise FileNotFoundError(f'YAML file does not exist, check path: {yaml_path}') + yaml_dict = parse_j2yaml(yaml_path, data) + + # yaml_dict is in the form {defaults: {key1: val1, ...}, base: {key1: val1, ...}, ...} + # _update_defaults replaces any keys/values in defaults with matching keys in base + yaml_dict = _update_defaults(yaml_dict) + # Override the YAML defaults with the host-specific capabilities # First update config.base edit_baseconfig(host, inputs, yaml_dict) - # loop over other configs and update them + # Update stage config + stage_dict = { + "@ICSDIR@": inputs.icsdir + } + host_dict = get_template_dict(host.info) + stage_dict = dict(stage_dict, **host_dict) + stage_input = f'{inputs.configdir}/config.stage_ic' + stage_output = f'{inputs.expdir}/{inputs.pslot}/config.stage_ic' + edit_config(stage_input, stage_output, host_dict, stage_dict) + + # Loop over other configs and update them with defaults for cfg in yaml_dict.keys(): if cfg == 'base': continue cfg_file = f'{inputs.expdir}/{inputs.pslot}/config.{cfg}' cfg_dict = get_template_dict(yaml_dict[cfg]) - edit_config(cfg_file, cfg_file, cfg_dict) + edit_config(cfg_file, cfg_file, host_dict, cfg_dict) return def edit_baseconfig(host, inputs, yaml_dict): """ - Parses and populates the templated `HOMEgfs/parm/config//config.base` + Parses and populates the templated `HOMEgfs/parm/config//config.base` to `EXPDIR/pslot/config.base` """ - tmpl_dict = { + # Create base_dict which holds templated variables to be written to config.base + base_dict = { "@HOMEgfs@": _top, "@MACHINE@": host.machine.upper()} - # Replace host related items - extend_dict = get_template_dict(host.info) - tmpl_dict = dict(tmpl_dict, **extend_dict) - if inputs.start in ["warm"]: is_warm_start = ".true." elif inputs.start in ["cold"]: is_warm_start = ".false." + else: + raise ValueError(f"Invalid start type: {inputs.start}") - extend_dict = dict() + # Construct a dictionary from user inputs extend_dict = { "@PSLOT@": inputs.pslot, "@SDATE@": datetime_to_YMDH(inputs.idate), @@ -335,39 +127,42 @@ def edit_baseconfig(host, inputs, yaml_dict): "@COMROOT@": inputs.comroot, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, - "@gfs_cyc@": inputs.gfs_cyc, + "@INTERVAL_GFS@": inputs.interval, + "@SDATE_GFS@": datetime_to_YMDH(inputs.sdate_gfs), "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } - tmpl_dict = dict(tmpl_dict, **extend_dict) - extend_dict = dict() if getattr(inputs, 'nens', 0) > 0: - extend_dict = { - "@CASEENS@": f'C{inputs.resensatmos}', - } - tmpl_dict = dict(tmpl_dict, **extend_dict) + extend_dict['@CASEENS@'] = f'C{inputs.resensatmos}' - extend_dict = dict() if inputs.mode in ['cycled']: - extend_dict = { - "@DOHYBVAR@": "YES" if inputs.nens > 0 else "NO", - } - tmpl_dict = dict(tmpl_dict, **extend_dict) + extend_dict["@DOHYBVAR@"] = "YES" if inputs.nens > 0 else "NO" - try: - tmpl_dict = dict(tmpl_dict, **get_template_dict(yaml_dict['base'])) - except KeyError: - pass + # Further extend/redefine base_dict with extend_dict + base_dict = dict(base_dict, **extend_dict) + + # Add/override 'base'-specific declarations in base_dict + if 'base' in yaml_dict: + base_dict = dict(base_dict, **get_template_dict(yaml_dict['base'])) base_input = f'{inputs.configdir}/config.base' base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' - edit_config(base_input, base_output, tmpl_dict) + edit_config(base_input, base_output, host.info, base_dict) return -def edit_config(input_config, output_config, config_dict): +def edit_config(input_config, output_config, host_info, config_dict): + """ + Given a templated input_config filename, parse it based on config_dict and + host_info and write it out to the output_config filename. + """ + + # Override defaults with machine-specific capabilties + # e.g. some machines are not able to run metp jobs + host_dict = get_template_dict(host_info) + config_dict = dict(config_dict, **host_dict) # Read input config with open(input_config, 'rt') as fi: @@ -391,9 +186,17 @@ def edit_config(input_config, output_config, config_dict): def get_template_dict(input_dict): + # Reads a templated input dictionary and updates the output + output_dict = dict() + for key, value in input_dict.items(): - output_dict[f'@{key}@'] = value + # In some cases, the same config may be templated twice + # Prevent adding additional "@"s + if "@" in key: + output_dict[f'{key}'] = value + else: + output_dict[f'@{key}@'] = value return output_dict @@ -405,6 +208,19 @@ def input_args(*argv): ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] + def _validate_interval(interval_str): + err_msg = f'must be a non-negative integer multiple of 6 ({interval_str} given)' + try: + interval = int(interval_str) + except ValueError: + raise ArgumentTypeError(err_msg) + + # This assumes the gdas frequency (assim_freq) is 6h + # If this changes, the modulus needs to as well + if interval < 0 or interval % 6 != 0: + raise ArgumentTypeError(err_msg) + return interval + def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') @@ -418,7 +234,10 @@ def _common_args(parser): type=str, required=False, default=os.getenv('HOME')) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) - parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--edate', help='end date experiment', required=False, type=lambda dd: to_datetime(dd)) + parser.add_argument('--account', help='HPC account to use; default is host-dependent', required=False, default=os.getenv('HPC_ACCOUNT')) + parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6', type=_validate_interval, required=False, default=6) + parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) return parser @@ -435,25 +254,21 @@ def _gfs_args(parser): return parser def _gfs_cycled_args(parser): - parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False, default=None) parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, - choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser - def _gfs_or_gefs_ensemble_args(parser): + def _any_ensemble_args(parser): parser.add_argument('--resensatmos', help='atmosphere resolution of the ensemble model forecast', type=int, required=False, default=192) parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20) return parser - def _gfs_or_gefs_forecast_args(parser): + def _any_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, - choices=[1, 2, 4], default=1, required=False) return parser def _gefs_args(parser): @@ -463,8 +278,15 @@ def _gefs_args(parser): default=os.path.join(_top, 'parm/config/gefs')) parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False, default=os.path.join(_top, 'parm/config/gefs/yaml/defaults.yaml')) - parser.add_argument('--icsdir', help='full path to initial condition directory [temporary hack in place for testing]', - type=str, required=False, default=None) + return parser + + def _sfs_args(parser): + parser.add_argument('--start', help='restart mode: warm or cold', type=str, + choices=['warm', 'cold'], required=False, default='cold') + parser.add_argument('--configdir', help=SUPPRESS, type=str, required=False, + default=os.path.join(_top, 'parm/config/sfs')) + parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False, + default=os.path.join(_top, 'parm/config/sfs/yaml/defaults.yaml')) return parser description = """ @@ -480,6 +302,7 @@ def _gefs_args(parser): sysparser = parser.add_subparsers(dest='system') gfs = sysparser.add_parser('gfs', help='arguments for GFS') gefs = sysparser.add_parser('gefs', help='arguments for GEFS') + sfs = sysparser.add_parser('sfs', help='arguments for SFS') gfsmodeparser = gfs.add_subparsers(dest='mode') gfscycled = gfsmodeparser.add_parser('cycled', help='arguments for cycled mode') @@ -488,8 +311,11 @@ def _gefs_args(parser): gefsmodeparser = gefs.add_subparsers(dest='mode') gefsforecasts = gefsmodeparser.add_parser('forecast-only', help='arguments for forecast-only mode') + sfsmodeparser = sfs.add_subparsers(dest='mode') + sfsforecasts = sfsmodeparser.add_parser('forecast-only', help='arguments for forecast-only mode') + # Common arguments across all modes - for subp in [gfscycled, gfsforecasts, gefsforecasts]: + for subp in [gfscycled, gfsforecasts, gefsforecasts, sfsforecasts]: subp = _common_args(subp) # GFS-only arguments @@ -497,12 +323,12 @@ def _gefs_args(parser): subp = _gfs_args(subp) # ensemble-only arguments - for subp in [gfscycled, gefsforecasts]: - subp = _gfs_or_gefs_ensemble_args(subp) + for subp in [gfscycled, gefsforecasts, sfsforecasts]: + subp = _any_ensemble_args(subp) # GFS/GEFS forecast-only additional arguments - for subp in [gfsforecasts, gefsforecasts]: - subp = _gfs_or_gefs_forecast_args(subp) + for subp in [gfsforecasts, gefsforecasts, sfsforecasts]: + subp = _any_forecast_args(subp) # cycled mode additional arguments for subp in [gfscycled]: @@ -512,7 +338,32 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) - return parser.parse_args(list(*argv) if len(argv) else None) + # SFS arguments + for subp in [sfsforecasts]: + subp = _sfs_args(subp) + + inputs = parser.parse_args(list(*argv) if len(argv) else None) + + # Validate dates + if inputs.edate is None: + inputs.edate = inputs.idate + + if inputs.edate < inputs.idate: + raise ArgumentTypeError(f'edate ({inputs.edate}) cannot be before idate ({inputs.idate})') + + # For forecast-only, GFS starts in the first cycle + if not hasattr(inputs, 'sdate_gfs'): + inputs.sdate_gfs = inputs.idate + + # For cycled, GFS starts after the half-cycle + if inputs.sdate_gfs is None: + inputs.sdate_gfs = inputs.idate + to_timedelta("6H") + + if inputs.interval > 0: + if inputs.sdate_gfs < inputs.idate or inputs.sdate_gfs > inputs.edate: + raise ArgumentTypeError(f'sdate_gfs ({inputs.sdate_gfs}) must be between idate ({inputs.idate}) and edate ({inputs.edate})') + + return inputs def query_and_clean(dirname, force_clean=False): @@ -524,7 +375,7 @@ def query_and_clean(dirname, force_clean=False): if os.path.exists(dirname): print(f'\ndirectory already exists in {dirname}') if force_clean: - overwrite = True + overwrite = "YES" print(f'removing directory ........ {dirname}\n') else: overwrite = input('Do you wish to over-write [y/N]: ') @@ -570,6 +421,10 @@ def main(*argv): validate_user_request(host, user_inputs) + # Update the default host account if the user supplied one + if user_inputs.account is not None: + host.info.ACCOUNT = user_inputs.account + # Determine ocean resolution if not provided if user_inputs.resdetocean <= 0: user_inputs.resdetocean = get_ocean_resolution(user_inputs.resdetatmos) @@ -582,11 +437,10 @@ def main(*argv): if create_rotdir: makedirs_if_missing(rotdir) - fill_ROTDIR(host, user_inputs) if create_expdir: makedirs_if_missing(expdir) - fill_EXPDIR(user_inputs) + fill_expdir(user_inputs) update_configs(host, user_inputs) print(f"*" * 100) diff --git a/workflow/tests/__init__.py b/workflow/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/workflow/tests/test_configuration.py b/workflow/tests/test_configuration.py new file mode 100644 index 00000000000..32e40c67af0 --- /dev/null +++ b/workflow/tests/test_configuration.py @@ -0,0 +1,32 @@ +import sys +from wxflow import Configuration + + +expdir = sys.argv[1] + +cfg = Configuration(expdir) + +print(f'experiment dir: {cfg.config_dir}') + +print('configuration files ...') +line_separator = '\n' # \escapes are not allowed inside f-strings +print(f'{line_separator.join(cfg.config_files)}') + +print(f'config.base: {cfg.find_config("config.base")}') + +print('*' * 80) +print('config.base ...') +base = cfg.parse_config('config.base') +cfg.print_config('config.base') +print(type(base)) +print(base.HOMEgfs) + +print('*' * 80) +print('config.anal...') +cfg.print_config(['config.base', 'config.anal']) + + +print('*' * 80) +print('config.efcs ...') +configs = ['config.base', 'config.fcst', 'config.efcs'] +cfg.print_config(configs) diff --git a/workflow/tests/test_hosts.py b/workflow/tests/test_hosts.py new file mode 100644 index 00000000000..b9fa969e143 --- /dev/null +++ b/workflow/tests/test_hosts.py @@ -0,0 +1,17 @@ +from hosts import Host + +print(f'supported hosts are: {", ".join(Host.SUPPORTED_HOSTS)}') + +print(f'host detected as: {Host.detect}') +print(f'scheduler on host: {Host().scheduler}') + +print('initializing host ...') +host = Host() + +print(f'hostname: {host.machine}') + +print(f'scheduler on host: {host.scheduler}') + +print('host information ...') +line_separator = '\n' # \escapes are not allowed inside f-strings +print(f'{line_separator.join(f"{key}: {host.info[key]}" for key in host.info.keys())}') diff --git a/workflow/tests/test_tasks.py b/workflow/tests/test_tasks.py new file mode 100644 index 00000000000..7e7e7eb5e7b --- /dev/null +++ b/workflow/tests/test_tasks.py @@ -0,0 +1,84 @@ +from rocoto.tasks import Tasks + + +class TestTasks: + + ''' + Tasks class tests + + Note: this is currently only testing a small fraction of the class. + ''' + + def test_job_groups(self): + test_array = list(range(0, 24)) + + # Test simple splitting with no breakpoints + test_groups = [{'fhrs': [0, 1, 2, 3, 4, 5], 'seg': 0}, + {'fhrs': [6, 7, 8, 9, 10, 11], 'seg': 0}, + {'fhrs': [12, 13, 14, 15, 16, 17], 'seg': 0}, + {'fhrs': [18, 19, 20, 21, 22, 23], 'seg': 0}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=4) == test_groups + + # Test with a break point that aligns with normal split point + test_groups = [{'fhrs': [0, 1, 2, 3, 4, 5], 'seg': 0}, + {'fhrs': [6, 7, 8, 9, 10, 11], 'seg': 0}, + {'fhrs': [12, 13, 14, 15, 16, 17], 'seg': 1}, + {'fhrs': [18, 19, 20, 21, 22, 23], 'seg': 1}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=4, breakpoints=[11]) == test_groups + + # Test with a break point not at a normal split point + test_groups = [{'fhrs': [0, 1, 2, 3, 4, 5, 6, 7], 'seg': 0}, + {'fhrs': [8, 9, 10, 11, 12, 13, 14], 'seg': 0}, + {'fhrs': [15, 16, 17, 18, 19], 'seg': 1}, + {'fhrs': [20, 21, 22, 23], 'seg': 1}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=4, breakpoints=[14]) == test_groups + + # Test highly skewed break point + test_groups = [{'fhrs': [0, 1, 2, 3, 4, 5, 6, 7], 'seg': 0}, + {'fhrs': [8, 9, 10, 11, 12, 13, 14, 15], 'seg': 0}, + {'fhrs': [16, 17, 18, 19, 20, 21, 22], 'seg': 0}, + {'fhrs': [23], 'seg': 1}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=4, breakpoints=[22]) == test_groups + + # Test with two break points that align + test_groups = [{'fhrs': [0, 1, 2, 3, 4, 5], 'seg': 0}, + {'fhrs': [6, 7, 8, 9, 10, 11], 'seg': 0}, + {'fhrs': [12, 13, 14, 15, 16, 17], 'seg': 1}, + {'fhrs': [18, 19, 20, 21, 22, 23], 'seg': 2}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=4, breakpoints=[11, 17]) == test_groups + + # Test with two skewed break points + test_groups = [{'fhrs': [0, 1], 'seg': 0}, + {'fhrs': [2, 3, 4, 5, 6, 7], 'seg': 1}, + {'fhrs': [8, 9, 10, 11, 12], 'seg': 1}, + {'fhrs': [13, 14, 15, 16, 17], 'seg': 1}, + {'fhrs': [18, 19, 20, 21, 22], 'seg': 1}, + {'fhrs': [23], 'seg': 2}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=6, breakpoints=[1, 22]) == test_groups + + # Test slightly irregular break points + test_groups = [{'fhrs': [0, 1, 2, 3], 'seg': 0}, + {'fhrs': [4, 5, 6], 'seg': 0}, + {'fhrs': [7, 8, 9, 10], 'seg': 1}, + {'fhrs': [11, 12, 13, 14], 'seg': 1}, + {'fhrs': [15, 16, 17, 18], 'seg': 1}, + {'fhrs': [19, 20, 21, 22, 23], 'seg': 2}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=6, breakpoints=[6, 18]) == test_groups + + # Test more groups than fhrs available + test_array = list(range(0, 6)) + test_groups = [{'fhrs': [0], 'seg': 0}, + {'fhrs': [1], 'seg': 0}, + {'fhrs': [2], 'seg': 0}, + {'fhrs': [3], 'seg': 0}, + {'fhrs': [4], 'seg': 0}, + {'fhrs': [5], 'seg': 0}] + assert Tasks.get_job_groups(fhrs=test_array, ngroups=15) == test_groups + + def test_multiply_HMS(self): + assert Tasks.multiply_HMS('00:10:00', 2) == '00:20:00' + assert Tasks.multiply_HMS('00:30:00', 10) == '05:00:00' + assert Tasks.multiply_HMS('01:15:00', 4) == '05:00:00' + assert Tasks.multiply_HMS('00:05:00', 1.5) == '00:07:30' + assert Tasks.multiply_HMS('00:40:00', 2.5) == '01:40:00' + assert Tasks.multiply_HMS('00:10:00', 1) == '00:10:00' diff --git a/workflow/wxflow b/workflow/wxflow new file mode 120000 index 00000000000..7ea96a12bfc --- /dev/null +++ b/workflow/wxflow @@ -0,0 +1 @@ +../sorc/wxflow/src/wxflow \ No newline at end of file From 75705e410525702730ad8fbbd02f033a2432264d Mon Sep 17 00:00:00 2001 From: "kayee.wong" Date: Thu, 6 Mar 2025 22:49:05 +0000 Subject: [PATCH 2/2] Realtime fixes --- jobs/rocoto/init.sh | 4 ++-- parm/config/gfs/config.base | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 32656fafc46..e1236c23b1a 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -21,8 +21,8 @@ status=$? [[ $status -ne 0 ]] && exit $status #JKH load ufs-utils modules -module use /scratch2/BMC/gsd-fv3-dev/KaYee.Wong/modulefiles -module load build.hera.gfsinit +module use /scratch1/BMC/gsd-fv3/rtruns/modulefiles +module load build.hera.2025Feb.gfsinit module list ############################################################### diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index d0abadc50d0..5c78778ccb4 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -130,6 +130,8 @@ export DUMP_SUFFIX="" if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi +#KYW for real time init task +export DATAROOT="${STMP}/RUNDIRS/${PSLOT}" # TODO: set via prod_envir in Ops export ARCDIR="${NOSCRUB}/archive/${PSLOT}" export ATARDIR="@ATARDIR@" export FETCHDIR="@FETCHDIR@"