From 2a6affdadba06f801cb927c91f98c9ff87d05b21 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 17 Oct 2022 16:05:36 +0000 Subject: [PATCH 1/9] Move fixed-file mapping variables out of config_defaults. Also, move other variables that are unlikely/never changed by user. --- parm/fixed_files_mapping.yaml | 208 ++++++++++++++++++++++++++ scripts/exregional_run_fcst.sh | 2 +- ush/config_defaults.yaml | 201 +++---------------------- ush/generate_FV3LAM_wflow.py | 2 +- ush/set_FV3nml_sfc_climo_filenames.py | 36 +---- ush/set_extrn_mdl_params.py | 49 ------ ush/set_ozone_param.py | 28 ++-- ush/set_thompson_mp_fix_files.py | 32 ++-- ush/setup.py | 80 ++++------ 9 files changed, 288 insertions(+), 350 deletions(-) create mode 100644 parm/fixed_files_mapping.yaml delete mode 100644 ush/set_extrn_mdl_params.py diff --git a/parm/fixed_files_mapping.yaml b/parm/fixed_files_mapping.yaml new file mode 100644 index 0000000000..2439e535a0 --- /dev/null +++ b/parm/fixed_files_mapping.yaml @@ -0,0 +1,208 @@ +fixed_files: + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to surface climatology + # files (on the native FV3-LAM grid) in the FIXlam directory derived from + # the corresponding surface climatology fields (the second column of the + # array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ + "FNALBC | snowfree_albedo", + "FNALBC2 | facsf", + "FNTG3C | substrate_temperature", + "FNVEGC | vegetation_greenness", + "FNVETC | vegetation_type", + "FNSOTC | soil_type", + "FNVMNC | vegetation_greenness", + "FNVMXC | vegetation_greenness", + "FNSLPC | slope_type", + "FNABSC | maximum_snow_albedo" + ] + + # + #----------------------------------------------------------------------- + # + # Set the array parameter containing the names of all the fields that the + # MAKE_SFC_CLIMO_TN task generates on the native FV3-LAM grid. + # + #----------------------------------------------------------------------- + # + SFC_CLIMO_FIELDS: [ + "facsf", + "maximum_snow_albedo", + "slope_type", + "snowfree_albedo", + "soil_type", + "substrate_temperature", + "vegetation_greenness", + "vegetation_type", + ] + + # + #----------------------------------------------------------------------- + # + # FNGLAC, ..., FNMSKH: + # Names of (some of the) global data files that are assumed to exist in + # a system directory specified (this directory is machine-dependent; + # the experiment generation scripts will set it and store it in the + # variable FIXgsm). These file names also appear directly in the forecast + # model's input namelist file. + # + #----------------------------------------------------------------------- + # + FNGLAC: &FNGLAC "global_glacier.2x2.grb" + FNMXIC: &FNMXIC "global_maxice.2x2.grb" + FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" + FNSNOC: &FNSNOC "global_snoclim.1.875.grb" + FNZORC: &FNZORC "igbp" + FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" + FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" + FNMSKH: &FNMSKH "seaice_newland.grb" + # + #----------------------------------------------------------------------- + # + # FIXgsm_FILES_TO_COPY_TO_FIXam: + # If not running in NCO mode, this array contains the names of the files + # to copy from the FIXgsm system directory to the FIXam directory under + # the experiment directory. Note that the last element has a dummy value. + # This last element will get reset by the workflow generation scripts to + # the name of the ozone production/loss file to copy from FIXgsm. The + # name of this file depends on the ozone parameterization being used, + # and that in turn depends on the CCPP physics suite specified for the + # experiment. Thus, the CCPP physics suite XML must first be read in to + # determine the ozone parameterizaton and then the name of the ozone + # production/loss file. These steps are carried out elsewhere (in one + # of the workflow generation scripts/functions). + # + #----------------------------------------------------------------------- + # + FIXgsm_FILES_TO_COPY_TO_FIXam: [ + *FNGLAC, + *FNMXIC, + *FNTSFC, + *FNSNOC, + *FNAISC, + *FNSMCC, + *FNMSKH, + "global_climaeropac_global.txt", + "fix_co2_proj/global_co2historicaldata_2010.txt", + "fix_co2_proj/global_co2historicaldata_2011.txt", + "fix_co2_proj/global_co2historicaldata_2012.txt", + "fix_co2_proj/global_co2historicaldata_2013.txt", + "fix_co2_proj/global_co2historicaldata_2014.txt", + "fix_co2_proj/global_co2historicaldata_2015.txt", + "fix_co2_proj/global_co2historicaldata_2016.txt", + "fix_co2_proj/global_co2historicaldata_2017.txt", + "fix_co2_proj/global_co2historicaldata_2018.txt", + "fix_co2_proj/global_co2historicaldata_2019.txt", + "fix_co2_proj/global_co2historicaldata_2020.txt", + "fix_co2_proj/global_co2historicaldata_2021.txt", + "global_co2historicaldata_glob.txt", + "co2monthlycyc.txt", + "global_h2o_pltc.f77", + "global_hyblev.l65.txt", + "global_zorclim.1x1.grb", + "global_sfc_emissivity_idx.txt", + "global_tg3clim.2.6x1.5.grb", + "global_solarconstant_noaa_an.txt", + "global_albedo4.1x1.grb", + "geo_em.d01.lat-lon.2.5m.HGT_M.nc", + "HGT.Beljaars_filtered.lat-lon.30s_res.nc", + "replace_with_FIXgsm_ozone_prodloss_filename" + ] + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to these files in + # the FIXam directory derived from the corresponding workflow variables + # containing file names (the second column of the array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ + !join_str ["FNGLAC | ",*FNGLAC], + !join_str ["FNMXIC | ",*FNMXIC], + !join_str ["FNTSFC | ",*FNTSFC], + !join_str ["FNSNOC | ",*FNSNOC], + !join_str ["FNAISC | ",*FNAISC], + !join_str ["FNSMCC | ",*FNSMCC], + !join_str ["FNMSKH | ",*FNMSKH] + ] + #"FNZORC | $FNZORC", + + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to surface climatology + # files (on the native FV3-LAM grid) in the FIXlam directory derived from + # the corresponding surface climatology fields (the second column of the + # array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ + "FNALBC | snowfree_albedo", + "FNALBC2 | facsf", + "FNTG3C | substrate_temperature", + "FNVEGC | vegetation_greenness", + "FNVETC | vegetation_type", + "FNSOTC | soil_type", + "FNVMNC | vegetation_greenness", + "FNVMXC | vegetation_greenness", + "FNSLPC | slope_type", + "FNABSC | maximum_snow_albedo" + ] + + + # + #----------------------------------------------------------------------- + # + # CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: + # This array specifies the mapping to use between the symlinks that need + # to be created in each cycle directory (these are the "files" that FV3 + # looks for) and their targets in the FIXam directory. The first column + # of the array specifies the symlink to be created, and the second column + # specifies its target file in FIXam (where columns are delineated by the + # pipe symbol "|"). + # + #----------------------------------------------------------------------- + # + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ + "aerosol.dat | global_climaeropac_global.txt", + "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", + "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", + "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", + "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", + "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", + "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", + "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", + "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", + "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", + "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", + "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", + "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", + "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", + "co2monthlycyc.txt | co2monthlycyc.txt", + "global_h2oprdlos.f77 | global_h2o_pltc.f77", + "global_albedo4.1x1.grb | global_albedo4.1x1.grb", + "global_zorclim.1x1.grb | global_zorclim.1x1.grb", + "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", + "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", + "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", + "global_o3prdlos.f77 | " + ] diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 008cfbd063..626a855ba5 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -290,7 +290,7 @@ static) files in the FIXam directory: # isn't really an advantage to using relative symlinks, so we use symlinks # with absolute paths. # -if [ "${RUN_ENVIR}" != "nco" ]; then +if [ "${SYMLINK_FIX_FILES}" == "FALSE" ]; then relative_link_flag="TRUE" else relative_link_flag="FALSE" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 2c5831a4df..43d29abe2f 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -38,7 +38,6 @@ user: # #----------------------------------------------------------------------- # - # mach_doc_start # Set machine and queue parameters. Definitions: # # MACHINE: @@ -46,7 +45,7 @@ user: # supported platform, and you want to use the Rocoto workflow manager, # you will need set MACHINE: "linux" and WORKFLOW_MANAGER: "rocoto". This # combination will assume a Slurm batch manager when generating the XML. - # Please see ush/valid_param_vals.sh for a full list of supported + # Please see ush/valid_param_vals.yaml for a full list of supported # platforms. # # ACCOUNT: @@ -72,13 +71,8 @@ platform: # the XML. Valid options: "rocoto" or "none" # # NCORES_PER_NODE: - # The number of cores available per node on the compute platform. Set - # for supported platforms in setup.sh, but is now also configurable for - # all platforms. - # - # LMOD_PATH: - # Path to the LMOD sh file on your Linux system. Is set automatically - # for supported machines. + # The number of cores available per node on the compute platform, now + # configurable for all platforms. # # BUILD_MOD_FN: # Name of alternative build module file to use if using an @@ -147,13 +141,10 @@ platform: # If this is not set or set to an empty string, it will be (re)set to a # machine-dependent value. # - # mach_doc_end - # #----------------------------------------------------------------------- # WORKFLOW_MANAGER: "" NCORES_PER_NODE: "" - LMOD_PATH: "" BUILD_MOD_FN: "" WFLOW_MOD_FN: "" BUILD_VER_FN: "" @@ -177,14 +168,15 @@ platform: # will run without MPI. # # RUN_CMD_FCST: - # The run command for the model forecast step. This will be appended to - # the end of the variable definitions file, so it can reference other - # variables. + # The run command for the model forecast step. # # RUN_CMD_POST: # The run command for post-processing (UPP). Can be left blank for smaller # domains, in which case UPP will run without MPI. # + # RUN_CMD_SERIAL: + # The run command for some serial jobs + # #----------------------------------------------------------------------- # RUN_CMD_SERIAL: "" @@ -211,6 +203,9 @@ platform: # MET_INSTALL_DIR: # Location to top-level directory of MET installation. # + # MET_BIN_EXEC: + # Subdirectory containing MET binaries e.g. "bin" + # # METPLUS_PATH: # Location to top-level directory of METplus installation. # @@ -328,7 +323,7 @@ platform: PRE_TASK_CMDS: "" # #----------------------------------------------------------------------- - # Test directories + # Test directories used in run_WE2E script #----------------------------------------------------------------------- # TEST_EXTRN_MDL_SOURCE_BASEDIR: "" @@ -340,14 +335,6 @@ platform: # WORKFLOW config parameters #----------------------------- workflow: - # - #----------------------------------------------------------------------- - # - # Unique ID for workflow run that will be set in setup.py - # - #----------------------------------------------------------------------- - # - WORKFLOW_ID: "" # #----------------------------------------------------------------------- # @@ -370,7 +357,6 @@ workflow: # #----------------------------------------------------------------------- # - # dir_doc_start # Set directories. Definitions: # # EXPT_BASEDIR: @@ -388,8 +374,6 @@ workflow: # This cannot be empty. If set to a null string here, it must be set to # a (non-empty) value in the user-defined experiment configuration file. # - # dir_doc_end - # # EXEC_SUBDIR: # The name of the subdirectory of ufs-srweather-app where executables are # installed. @@ -1042,7 +1026,7 @@ task_make_grid: # in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; # they are not needed by the forecast model. # NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified - # variable. Just set it in the function set_gridparams_ESGgrid.sh. + # variable. Just set it in the function set_gridparams_ESGgrid.py. # # Note that: # @@ -1263,8 +1247,7 @@ task_get_extrn_lbcs: # should start. For example, the forecast should use lateral boundary # conditions from the GFS started 6 hours earlier, then # EXTRN_MDL_LBCS_OFFSET_HRS=6. - # Note: the default value is model-dependent and set in - # set_extrn_mdl_params.sh + # Note: the default value is model-dependent and set in setup.py # # FV3GFS_FILE_FMT_LBCS: # If using the FV3GFS model as the source of the LBCs (i.e. if @@ -1379,7 +1362,6 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" - LBC_SPEC_FCST_HRS: [] #---------------------------- # FORECAST config parameters @@ -1387,7 +1369,7 @@ task_make_lbcs: task_run_fcst: RUN_FCST_TN: "run_fcst" NNODES_RUN_FCST: "" # This is calculated in the workflow generation scripts, so no need to set here. - PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.sh + PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.py WTIME_RUN_FCST: 04:30:00 MAXTRIES_RUN_FCST: 1 # @@ -1576,7 +1558,7 @@ task_run_fcst: # commonly used set of grid-dependent parameters. The predefined grid # parameters are specified in the script # - # $HOMEdir/ush/set_predef_grid_params.sh + # $HOMEdir/ush/set_predef_grid_params.py # #----------------------------------------------------------------------- # @@ -1613,6 +1595,9 @@ task_run_fcst: # # Set parameters associated with the fixed (i.e. static) files. Definitions: # + # SYMLINK_FIX_FILES: + # Symlink fix files to experiment directory if true, otherwise copy the files + # # FIXgsm: # System directory in which the majority of fixed (i.e. time-independent) # files that are needed to run the FV3-LAM model are located @@ -1623,158 +1608,12 @@ task_run_fcst: # FIXlut: # System directory where the lookup tables for optics properties are located # - # FNGLAC, ..., FNMSKH: - # Names of (some of the) global data files that are assumed to exist in - # a system directory specified (this directory is machine-dependent; - # the experiment generation scripts will set it and store it in the - # variable FIXgsm). These file names also appear directly in the forecast - # model's input namelist file. - # - # FIXgsm_FILES_TO_COPY_TO_FIXam: - # If not running in NCO mode, this array contains the names of the files - # to copy from the FIXgsm system directory to the FIXam directory under - # the experiment directory. Note that the last element has a dummy value. - # This last element will get reset by the workflow generation scripts to - # the name of the ozone production/loss file to copy from FIXgsm. The - # name of this file depends on the ozone parameterization being used, - # and that in turn depends on the CCPP physics suite specified for the - # experiment. Thus, the CCPP physics suite XML must first be read in to - # determine the ozone parameterizaton and then the name of the ozone - # production/loss file. These steps are carried out elsewhere (in one - # of the workflow generation scripts/functions). - # - # FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to these files in - # the FIXam directory derived from the corresponding workflow variables - # containing file names (the second column of the array). - # - # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXlam directory derived from - # the corresponding surface climatology fields (the second column of the - # array). - # - # CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: - # This array specifies the mapping to use between the symlinks that need - # to be created in each cycle directory (these are the "files" that FV3 - # looks for) and their targets in the FIXam directory. The first column - # of the array specifies the symlink to be created, and the second column - # specifies its target file in FIXam (where columns are delineated by the - # pipe symbol "|"). - # - #----------------------------------------------------------------------- - # - # Because the default values are dependent on the platform, we set these - # to a null string which will then be overwritten in setup.sh unless the - # user has specified a different value in config.sh + #----------------------------------------------------------------------- # SYMLINK_FIX_FILES: true FIXgsm: "" FIXaer: "" FIXlut: "" - - FNGLAC: &FNGLAC "global_glacier.2x2.grb" - FNMXIC: &FNMXIC "global_maxice.2x2.grb" - FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" - FNSNOC: &FNSNOC "global_snoclim.1.875.grb" - FNZORC: &FNZORC "igbp" - FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" - FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" - FNMSKH: &FNMSKH "seaice_newland.grb" - - FIXgsm_FILES_TO_COPY_TO_FIXam: [ - *FNGLAC, - *FNMXIC, - *FNTSFC, - *FNSNOC, - *FNAISC, - *FNSMCC, - *FNMSKH, - "global_climaeropac_global.txt", - "fix_co2_proj/global_co2historicaldata_2010.txt", - "fix_co2_proj/global_co2historicaldata_2011.txt", - "fix_co2_proj/global_co2historicaldata_2012.txt", - "fix_co2_proj/global_co2historicaldata_2013.txt", - "fix_co2_proj/global_co2historicaldata_2014.txt", - "fix_co2_proj/global_co2historicaldata_2015.txt", - "fix_co2_proj/global_co2historicaldata_2016.txt", - "fix_co2_proj/global_co2historicaldata_2017.txt", - "fix_co2_proj/global_co2historicaldata_2018.txt", - "fix_co2_proj/global_co2historicaldata_2019.txt", - "fix_co2_proj/global_co2historicaldata_2020.txt", - "fix_co2_proj/global_co2historicaldata_2021.txt", - "global_co2historicaldata_glob.txt", - "co2monthlycyc.txt", - "global_h2o_pltc.f77", - "global_hyblev.l65.txt", - "global_zorclim.1x1.grb", - "global_sfc_emissivity_idx.txt", - "global_tg3clim.2.6x1.5.grb", - "global_solarconstant_noaa_an.txt", - "global_albedo4.1x1.grb", - "geo_em.d01.lat-lon.2.5m.HGT_M.nc", - "HGT.Beljaars_filtered.lat-lon.30s_res.nc", - "replace_with_FIXgsm_ozone_prodloss_filename" - ] - - # - # It is possible to remove this as a workflow variable and make it only - # a local one since it is used in only one script. - # - FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ - !join_str ["FNGLAC | ",*FNGLAC], - !join_str ["FNMXIC | ",*FNMXIC], - !join_str ["FNTSFC | ",*FNTSFC], - !join_str ["FNSNOC | ",*FNSNOC], - !join_str ["FNAISC | ",*FNAISC], - !join_str ["FNSMCC | ",*FNSMCC], - !join_str ["FNMSKH | ",*FNMSKH] - ] - #"FNZORC | $FNZORC", - - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo" - ] - - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ - "aerosol.dat | global_climaeropac_global.txt", - "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", - "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", - "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", - "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", - "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", - "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", - "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", - "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", - "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", - "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", - "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", - "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", - "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", - "co2monthlycyc.txt | co2monthlycyc.txt", - "global_h2oprdlos.f77 | global_h2o_pltc.f77", - "global_albedo4.1x1.grb | global_albedo4.1x1.grb", - "global_zorclim.1x1.grb | global_zorclim.1x1.grb", - "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", - "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", - "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", - "global_o3prdlos.f77 | " - ] #---------------------------- # POST config parameters @@ -2184,7 +2023,7 @@ global: # # Set default SPP stochastic physics options. Each SPP option is an array, # applicable (in order) to the scheme/parameter listed in SPP_VAR_LIST. - # Enter each value of the array in config.sh as shown below without commas + # Enter each value of the array in config.yaml as shown below without commas # or single quotes (e.g., SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ). # Both commas and single quotes will be added by Jinja when creating the # namelist. diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 11a86c9aaa..c696767862 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -992,7 +992,7 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> the rocotorun command must be issued immediately before issuing the rocotostat command. - For automatic resubmission of the workflow (say every 3 minutes), the + For automatic resubmission of the workflow (say every {CRON_RELAUNCH_INTVL_MNTS} minutes), the following line can be added to the user's crontab (use \"crontab -e\" to edit the cron table): diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index e94ad5b183..e713ebc2e0 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -17,6 +17,7 @@ rm_vrfy, import_vars, set_env_var, + load_config_file, load_shell_config, flatten_dict, define_macos_utilities, @@ -26,7 +27,6 @@ from set_namelist import set_namelist - def set_FV3nml_sfc_climo_filenames(): """ This function sets the values of the variables in @@ -46,6 +46,11 @@ def set_FV3nml_sfc_climo_filenames(): # import all environment variables import_vars() + # fixed file mapping variables + fixed_cfg = load_config_file(os.path.join(PARMdir,"fixed_files_mapping.yaml")) + IMPORTS = ["SFC_CLIMO_FIELDS", "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"] + import_vars(dictionary=flatten_dict(fixed_cfg), env_vars=IMPORTS) + # The regular expression regex_search set below will be used to extract # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING # the name of the namelist variable to set and the corresponding surface @@ -159,7 +164,7 @@ def setUp(self): os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - set_env_var("USHdir", USHdir) + set_env_var("PARMdir", PARMdir) set_env_var("EXPTDIR", EXPTDIR) set_env_var("FIXlam", FIXlam) set_env_var("DO_ENSEMBLE", False) @@ -167,30 +172,3 @@ def setUp(self): set_env_var("RUN_ENVIR", "nco") set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING = [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo", - ] - SFC_CLIMO_FIELDS = [ - "facsf", - "maximum_snow_albedo", - "slope_type", - "snowfree_albedo", - "soil_type", - "substrate_temperature", - "vegetation_greenness", - "vegetation_type", - ] - set_env_var( - "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING", - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING, - ) - set_env_var("SFC_CLIMO_FIELDS", SFC_CLIMO_FIELDS) diff --git a/ush/set_extrn_mdl_params.py b/ush/set_extrn_mdl_params.py deleted file mode 100644 index 7d52055031..0000000000 --- a/ush/set_extrn_mdl_params.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import unittest - -from python_utils import import_vars, export_vars, set_env_var, get_env_var - - -def set_extrn_mdl_params(): - """Sets parameters associated with the external model used for initial - conditions (ICs) and lateral boundary conditions (LBCs). - Args: - None - Returns: - None - """ - - # import all env variables - import_vars() - - global EXTRN_MDL_LBCS_OFFSET_HRS - - # - # ----------------------------------------------------------------------- - # - # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift - # the starting time of the external model that provides lateral boundary - # conditions. - # - # ----------------------------------------------------------------------- - # - if EXTRN_MDL_NAME_LBCS == "RAP": - EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "3" - else: - EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "0" - - # export values we set above - env_vars = ["EXTRN_MDL_LBCS_OFFSET_HRS"] - export_vars(env_vars=env_vars) - - -class Testing(unittest.TestCase): - def test_extrn_mdl_params(self): - set_extrn_mdl_params() - EXTRN_MDL_LBCS_OFFSET_HRS = get_env_var("EXTRN_MDL_LBCS_OFFSET_HRS") - self.assertEqual(EXTRN_MDL_LBCS_OFFSET_HRS, 3) - - def setUp(self): - set_env_var("EXTRN_MDL_NAME_LBCS", "RAP") - set_env_var("EXTRN_MDL_LBCS_OFFSET_HRS", None) diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index 5ed4449fe1..23c3e985e5 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -17,7 +17,8 @@ find_pattern_in_str, ) -def set_ozone_param(ccpp_phys_suite_fp): + +def set_ozone_param(ccpp_phys_suite_fp, CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, FIXgsm_FILES_TO_COPY_TO_FIXam, VERBOSE): """Function that does the following: (1) Determines the ozone parameterization being used by checking in the CCPP physics suite XML. @@ -41,15 +42,14 @@ def set_ozone_param(ccpp_phys_suite_fp): Args: ccpp_phys_suite_fp: full path to CCPP physics suite + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + FIXgsm_FILES_TO_COPY_TO_FIXam Returns: ozone_param: a string """ print_input_args(locals()) - # import all environment variables - import_vars() - # # ----------------------------------------------------------------------- # @@ -166,9 +166,6 @@ def set_ozone_param(ccpp_phys_suite_fp): fixgsm_ozone_fn_is_set = \"{fixgsm_ozone_fn_is_set}\"''' ) - EXPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - export_vars(env_vars=EXPORTS) - return ozone_param @@ -178,15 +175,15 @@ def test_set_ozone_param(self): self.assertEqual( "ozphys_2015", set_ozone_param( - ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" + f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE=True ), ) def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - + global CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ "aerosol.dat | global_climaeropac_global.txt", "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", @@ -209,6 +206,7 @@ def setUp(self): "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] + global FIXgsm_FILES_TO_COPY_TO_FIXam FIXgsm_FILES_TO_COPY_TO_FIXam = [ "global_glacier.2x2.grb", "global_maxice.2x2.grb", @@ -241,9 +239,3 @@ def setUp(self): "HGT.Beljaars_filtered.lat-lon.30s_res.nc", "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] - - set_env_var( - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - ) - set_env_var("FIXgsm_FILES_TO_COPY_TO_FIXam", FIXgsm_FILES_TO_COPY_TO_FIXam) diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 93dc3c5de6..40eb00bd9b 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -17,7 +17,8 @@ ) -def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): +def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, FIXgsm_FILES_TO_COPY_TO_FIXam): """Function that first checks whether the Thompson microphysics parameterization is being called by the selected physics suite. If not, it sets the output variable whose name is specified by @@ -31,14 +32,17 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): Args: ccpp_phys_suite_fp: full path to CCPP physics suite thompson_mp_climo_fn: netcdf file for thompson microphysics + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + FIXgsm_FILES_TO_COPY_TO_FIXam Returns: boolean: sdf_uses_thompson_mp """ print_input_args(locals()) - # import all environment variables - import_vars() + # import some environment variables + IMPORTS=["EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", "CCPP_PHYS_SUITE"] + import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- @@ -113,12 +117,6 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): """ ) - EXPORTS = [ - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - "FIXgsm_FILES_TO_COPY_TO_FIXam", - ] - export_vars(env_vars=EXPORTS) - return sdf_uses_thompson_mp @@ -128,19 +126,19 @@ def test_set_thompson_mp_fix_files(self): self.assertEqual( True, set_thompson_mp_fix_files( - ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", - thompson_mp_climo_fn="Thompson_MP_MONTHLY_CLIMO.nc", + f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", + "Thompson_MP_MONTHLY_CLIMO.nc", + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, ), ) def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) set_env_var("EXTRN_MDL_NAME_ICS", "FV3GFS") set_env_var("EXTRN_MDL_NAME_LBCS", "FV3GFS") set_env_var("CCPP_PHYS_SUITE", "FV3_GSD_SAR") + global CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ "aerosol.dat | global_climaeropac_global.txt", "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", @@ -164,6 +162,7 @@ def setUp(self): "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] + global FIXgsm_FILES_TO_COPY_TO_FIXam FIXgsm_FILES_TO_COPY_TO_FIXam = [ "global_glacier.2x2.grb", "global_maxice.2x2.grb", @@ -197,8 +196,3 @@ def setUp(self): "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] - set_env_var( - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - ) - set_env_var("FIXgsm_FILES_TO_COPY_TO_FIXam", FIXgsm_FILES_TO_COPY_TO_FIXam) diff --git a/ush/setup.py b/ush/setup.py index 3fa7313727..9a8569f570 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -32,7 +32,6 @@ from set_cycle_dates import set_cycle_dates from set_predef_grid_params import set_predef_grid_params from set_ozone_param import set_ozone_param -from set_extrn_mdl_params import set_extrn_mdl_params from set_gridparams_ESGgrid import set_gridparams_ESGgrid from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid from link_fix import link_fix @@ -180,6 +179,10 @@ def get_location(xcs,fmt): # make machine name uppercase MACHINE = uppercase(MACHINE) + # Load fixed-files mapping file + cfg_f = load_config_file(os.path.join(USHdir, os.pardir, "parm", "fixed_files_mapping.yaml")) + import_vars(dictionary=flatten_dict(cfg_f)) + # Load constants file and save its contents to a variable for later cfg_c = load_config_file(os.path.join(USHdir, CONSTANTS_FN)) import_vars(dictionary=flatten_dict(cfg_c)) @@ -194,6 +197,7 @@ def get_location(xcs,fmt): # global WORKFLOW_ID WORKFLOW_ID = "id_" + str(int(datetime.datetime.now().timestamp())) + cfg_d["workflow"]["WORKFLOW_ID"] = WORKFLOW_ID log_info(f"""WORKFLOW ID = {WORKFLOW_ID}""") # @@ -614,6 +618,7 @@ def get_location(xcs,fmt): LBC_SPEC_INTVL_HRS, LBC_SPEC_INTVL_HRS + FCST_LEN_HRS, LBC_SPEC_INTVL_HRS ) ] + cfg_d["task_make_lbcs"]["LBC_SPEC_FCST_HRS"] = LBC_SPEC_FCST_HRS # # ----------------------------------------------------------------------- # @@ -1022,13 +1027,13 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # - # export env vars before calling another module - export_vars() - - OZONE_PARAM = set_ozone_param(CCPP_PHYS_SUITE_IN_CCPP_FP) + OZONE_PARAM = set_ozone_param( + CCPP_PHYS_SUITE_IN_CCPP_FP, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE=VERBOSE, + ) - IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- # @@ -1288,21 +1293,21 @@ def get_location(xcs,fmt): else: SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") + # # ----------------------------------------------------------------------- # - # Set cycle-independent parameters associated with the external models - # from which we will obtain the ICs and LBCs. + # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift + # the starting time of the external model that provides lateral boundary + # conditions. # # ----------------------------------------------------------------------- # + global EXTRN_MDL_LBCS_OFFSET_HRS + if EXTRN_MDL_NAME_LBCS == "RAP": + EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "3" + else: + EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "0" - # export env vars before calling another module - export_vars() - - set_extrn_mdl_params() - - IMPORTS = ["EXTRN_MDL_LBCS_OFFSET_HRS"] - import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- # @@ -1579,48 +1584,19 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # SDF_USES_THOMPSON_MP = set_thompson_mp_fix_files( - ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP, - thompson_mp_climo_fn=THOMPSON_MP_CLIMO_FN, + CCPP_PHYS_SUITE_IN_CCPP_FP, + THOMPSON_MP_CLIMO_FN, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, ) - IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - import_vars(env_vars=IMPORTS) - - # - # ----------------------------------------------------------------------- - # - # Generate the shell script that will appear in the experiment directory - # (EXPTDIR) and will contain definitions of variables needed by the va- - # rious scripts in the workflow. We refer to this as the experiment/ - # workflow global variable definitions file. We will create this file - # by: - # - # 1) Copying the default workflow/experiment configuration file (speci- - # fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- - # rectory specified by USHdir) to the experiment directory and rena- - # ming it to the name specified by GLOBAL_VAR_DEFNS_FN. - # - # 2) Resetting the default variable values in this file to their current - # values. This is necessary because these variables may have been - # reset by the user-specified configuration file (if one exists in - # USHdir) and/or by this setup script, e.g. because predef_domain is - # set to a valid non-empty value. - # - # 3) Appending to the variable definitions file any new variables intro- - # duced in this setup script that may be needed by the scripts that - # perform the various tasks in the workflow (and which source the va- - # riable defintions file). - # - # First, set the full path to the variable definitions file and copy the - # default configuration script into it. - # - # ----------------------------------------------------------------------- - # - # global variable definition file path global GLOBAL_VAR_DEFNS_FP GLOBAL_VAR_DEFNS_FP = os.path.join(EXPTDIR, GLOBAL_VAR_DEFNS_FN) + # fixed files section + cfg_d.update(cfg_f) + # update dictionary with globals() values update_dict(globals(), cfg_d) From 13d55129474caf0565889bacdf7b1438a22ffc87 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 21 Oct 2022 20:37:43 +0000 Subject: [PATCH 2/9] Minor bug fix. --- ush/machine/odin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index 67a2fb49f1..38caa4bfc1 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -2,7 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm - DOMAIN_PREGEN_BASEDIR: /FV3LAM_pregen + DOMAIN_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: workq QUEUE_DEFAULT: workq PARTITION_FCST: workq From e64be4c7360c3d645c76f29621445f283a3729c3 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 22 Oct 2022 15:32:31 +0000 Subject: [PATCH 3/9] Use python script to check python version and packages. --- tests/WE2E/run_WE2E_tests.sh | 51 +++--------------------------------- ush/check_python_version.py | 50 +++++++++++++++++++++++++++++++++++ ush/generate_FV3LAM_wflow.py | 41 ++++------------------------- 3 files changed, 58 insertions(+), 84 deletions(-) create mode 100755 ush/check_python_version.py diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index ec06a88150..53edf57fde 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -69,56 +69,11 @@ WE2Edir="$TESTSdir/WE2E" # #----------------------------------------------------------------------- # - -# This line will return two numbers: the python major and minor versions -pyversion=($(/usr/bin/env python3 -c 'import platform; major, minor, patch = platform.python_version_tuple(); print(major); print(minor)')) - -#Now, set an error check variable so that we can print all python errors rather than just the first -pyerrors=0 - -# Check that the call to python3 returned no errors, then check if the -# python3 minor version is 6 or higher -if [[ -z "$pyversion" ]];then - print_info_msg "\ - - Error: python3 not found" - pyerrors=$((pyerrors+1)) -else - if [[ ${#pyversion[@]} -lt 2 ]]; then - print_info_msg "\ - - Error retrieving python3 version" - pyerrors=$((pyerrors+1)) - elif [[ ${pyversion[1]} -lt 6 ]]; then - print_info_msg "\ - - Error: python version must be 3.6 or higher - python version: ${pyversion[*]}" - pyerrors=$((pyerrors+1)) - fi +python3 $USHdir/check_python_version.py +if [[ $? -ne 0 ]]; then + exit 1 fi -#Next, check for the non-standard python packages: jinja2, yaml, and f90nml -pkgs=(jinja2 yaml f90nml) -for pkg in ${pkgs[@]} ; do - if ! /usr/bin/env python3 -c "import ${pkg}" &> /dev/null; then - print_info_msg "\ - - Error: python module ${pkg} not available" - pyerrors=$((pyerrors+1)) - fi -done - -#Finally, check if the number of errors is >0, and if so exit with helpful message -if [ $pyerrors -gt 0 ];then - print_err_msg_exit "\ - Errors found: check your python environment - - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - -" -fi # #----------------------------------------------------------------------- # diff --git a/ush/check_python_version.py b/ush/check_python_version.py new file mode 100755 index 0000000000..7b35b9fe3c --- /dev/null +++ b/ush/check_python_version.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import sys +import logging +import platform +from textwrap import dedent + +def check_python_version(): + """ Check if python version >= 3.6 and presence of some + non-standard packages currently jinja2, yaml, f90nml""" + + # Check for non-standard python packages + try: + import jinja2 + import yaml + import f90nml + except ImportError as error: + logging.error(dedent( + """ + Error: Missing python package required by the SRW app + """ + )) + raise + + + # check python version + major, minor, patch = platform.python_version_tuple() + if int(major) < 3 or int(minor) < 6: + logging.error(dedent( + f""" + Error: python version must be 3.6 or higher + Your python version is: {major}.{minor}""" + )) + raise Exception("Python version below 3.6") + +if __name__ == "__main__": + try: + check_python_version() + except: + logging.exception(dedent( + f""" + ************************************************************************* + FATAL ERROR: + The system does not meet minimum requirements for running the SRW app. + Instructions for setting up python environments can be found on the web: + https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started + *************************************************************************\n + """ + )) + sys.exit(1) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index c696767862..f55a6d8cbe 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -2,7 +2,6 @@ import os import sys -import platform import subprocess import unittest import logging @@ -37,30 +36,7 @@ from get_crontab_contents import add_crontab_line from fill_jinja_template import fill_jinja_template from set_namelist import set_namelist - - -def python_error_handler(): - """Error handler for missing packages""" - - print_err_msg_exit( - """ - Errors found: check your python environment - - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - """, - stack_trace=False, - ) - - -# Check for non-standard python packages -try: - import jinja2 - import yaml - import f90nml -except ImportError as error: - print_info_msg(error.__class__.__name__ + ": " + str(error)) - python_error_handler() +from check_python_version import check_python_version def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> None: @@ -77,6 +53,10 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # Set up logging to write to screen and logfile setup_logging(logfile) + # Check python version and presence of some non-standard packages + check_python_version() + + # Note start of workflow generation log_info( """ ======================================================================== @@ -84,17 +64,6 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> ========================================================================""" ) - # check python version - major, minor, patch = platform.python_version_tuple() - if int(major) < 3 or int(minor) < 6: - logging.error( - f""" - - Error: python version must be 3.6 or higher - python version: {major}.{minor}""" - ) - raise - # define utilities define_macos_utilities() From c52e1f53c6d8f8b71fea1869184eeeb1c030e915 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 22 Oct 2022 16:37:44 +0000 Subject: [PATCH 4/9] Minor fixes. --- ush/generate_FV3LAM_wflow.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index f55a6d8cbe..b7018e7133 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -902,21 +902,8 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # ----------------------------------------------------------------------- # cp_vrfy(os.path.join(USHdir, EXPT_CONFIG_FN), EXPTDIR) - # - # ----------------------------------------------------------------------- - # - # For convenience, print out the commands that need to be issued on the - # command line in order to launch the workflow and to check its status. - # Also, print out the line that should be placed in the user's cron table - # in order for the workflow to be continually resubmitted. - # - # ----------------------------------------------------------------------- - # - if WORKFLOW_MANAGER == "rocoto": - wflow_db_fn = f"{os.path.splitext(WFLOW_XML_FN)[0]}.db" - rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" - rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + # Note workflow generation completion log_info( f""" ======================================================================== @@ -930,13 +917,20 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> ======================================================================== """ ) + # # ----------------------------------------------------------------------- # - # If rocoto is required, print instructions on how to use it + # For convenience, print out the commands that need to be issued on the + # command line in order to launch the workflow and to check its status. + # Also, print out the line that should be placed in the user's cron table + # in order for the workflow to be continually resubmitted. # # ----------------------------------------------------------------------- # if WORKFLOW_MANAGER == "rocoto": + wflow_db_fn = f"{os.path.splitext(WFLOW_XML_FN)[0]}.db" + rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" log_info( f""" From 4c02e376ecc2c081080cd8c17098096a13a93992 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 22 Oct 2022 10:52:11 -0600 Subject: [PATCH 5/9] Format python files with black. --- ush/UFS_plot_domains.py | 6 +- ush/check_python_version.py | 39 +- ush/check_ruc_lsm.py | 4 +- ush/create_diag_table_file.py | 4 +- ush/generate_FV3LAM_wflow.py | 98 +++-- ush/get_crontab_contents.py | 16 +- .../check_for_preexist_dir_file.py | 19 +- ush/python_utils/check_var_valid_value.py | 1 + ush/python_utils/config_parser.py | 17 +- ush/python_utils/print_msg.py | 6 +- ush/retrieve_data.py | 3 +- ush/set_FV3nml_ens_stoch_seeds.py | 4 +- ush/set_FV3nml_sfc_climo_filenames.py | 4 +- ush/set_cycle_dates.py | 11 +- ush/set_gridparams_ESGgrid.py | 12 +- ush/set_gridparams_GFDLgrid.py | 2 +- ush/set_ozone_param.py | 27 +- ush/set_predef_grid_params.py | 7 +- ush/set_thompson_mp_fix_files.py | 13 +- ush/setup.py | 393 +++++++++++------- ush/test_retrieve_data.py | 97 ++--- 21 files changed, 472 insertions(+), 311 deletions(-) diff --git a/ush/UFS_plot_domains.py b/ush/UFS_plot_domains.py index d752c2696b..9068fa4163 100755 --- a/ush/UFS_plot_domains.py +++ b/ush/UFS_plot_domains.py @@ -128,7 +128,7 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): - #print("Hello from a function") + # print("Hello from a function") # This function takes the lambert domain we have defined, lambert_map, as well as # pps (the number of points to interpolate and draw for each side of the lambert "rectangle"), @@ -187,8 +187,8 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): # Need to replace final instruction with Path.CLOSEPOLY instructions[-1] = Path.CLOSEPOLY - #print("vertices=", vertices) - #print("instructions=", instructions) + # print("vertices=", vertices) + # print("instructions=", instructions) return vertices, instructions diff --git a/ush/check_python_version.py b/ush/check_python_version.py index 7b35b9fe3c..3888efdef2 100755 --- a/ush/check_python_version.py +++ b/ush/check_python_version.py @@ -5,9 +5,10 @@ import platform from textwrap import dedent + def check_python_version(): - """ Check if python version >= 3.6 and presence of some - non-standard packages currently jinja2, yaml, f90nml""" + """Check if python version >= 3.6 and presence of some + non-standard packages currently jinja2, yaml, f90nml""" # Check for non-standard python packages try: @@ -15,30 +16,35 @@ def check_python_version(): import yaml import f90nml except ImportError as error: - logging.error(dedent( - """ + logging.error( + dedent( + """ Error: Missing python package required by the SRW app """ - )) + ) + ) raise - # check python version major, minor, patch = platform.python_version_tuple() if int(major) < 3 or int(minor) < 6: - logging.error(dedent( - f""" + logging.error( + dedent( + f""" Error: python version must be 3.6 or higher Your python version is: {major}.{minor}""" - )) + ) + ) raise Exception("Python version below 3.6") - + + if __name__ == "__main__": - try: - check_python_version() - except: - logging.exception(dedent( - f""" + try: + check_python_version() + except: + logging.exception( + dedent( + f""" ************************************************************************* FATAL ERROR: The system does not meet minimum requirements for running the SRW app. @@ -46,5 +52,6 @@ def check_python_version(): https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started *************************************************************************\n """ - )) + ) + ) sys.exit(1) diff --git a/ush/check_ruc_lsm.py b/ush/check_ruc_lsm.py index 8d6e287df9..3afe1f8264 100644 --- a/ush/check_ruc_lsm.py +++ b/ush/check_ruc_lsm.py @@ -32,7 +32,9 @@ class Testing(unittest.TestCase): def test_check_ruc_lsm(self): USHdir = os.path.dirname(os.path.abspath(__file__)) self.assertTrue( - check_ruc_lsm(ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml") + check_ruc_lsm( + ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" + ) ) def setUp(self): diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index cd1943ef77..c939c6ee6b 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -133,9 +133,7 @@ def setUp(self): USHdir = os.path.dirname(os.path.abspath(__file__)) PARMdir = os.path.join(USHdir, "..", "parm") DIAG_TABLE_FN = "diag_table" - DIAG_TABLE_TMPL_FP = os.path.join( - PARMdir, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" - ) + DIAG_TABLE_TMPL_FP = os.path.join(PARMdir, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2") set_env_var("DEBUG", True) set_env_var("VERBOSE", True) set_env_var("USHdir", USHdir) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index b7018e7133..e5c26703d3 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -39,7 +39,7 @@ from check_python_version import check_python_version -def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> None: +def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> None: """Function to setup a forecast experiment and create a workflow (according to the parameters specified in the config file) @@ -58,16 +58,16 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # Note start of workflow generation log_info( - """ + """ ======================================================================== Starting experiment generation... ========================================================================""" ) - # define utilities + # define utilities define_macos_utilities() - # The setup function reads the user configuration file and fills in + # The setup function reads the user configuration file and fills in # non-user-specified values from config_defaults.yaml setup() @@ -396,13 +396,13 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> settings_str = cfg_to_yaml_str(settings) log_info( - f""" - The variable \"settings\" specifying values of the rococo XML variables - has been set as follows: - #----------------------------------------------------------------------- - settings =\n\n""", - verbose=VERBOSE, - ) + f""" + The variable \"settings\" specifying values of the rococo XML variables + has been set as follows: + #----------------------------------------------------------------------- + settings =\n\n""", + verbose=VERBOSE, + ) log_info(settings_str, verbose=VERBOSE) # @@ -817,7 +817,7 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> settings_str = cfg_to_yaml_str(settings) log_info( - f""" + f""" The variable \"settings\" specifying values of the weather model's namelist variables has been set as follows:\n""", verbose=VERBOSE, @@ -890,7 +890,15 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> if NOMADS: raise Exception("Nomads script does not work!") - # get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS) + # get_nomads_data( + # NOMADS_file_type, + # EXPTDIR, + # USHdir, + # DATE_FIRST_CYCL, + # CYCL_HRS, + # FCST_LEN_HRS, + # LBC_SPEC_INTVL_HRS, + # ) # # ----------------------------------------------------------------------- @@ -966,41 +974,61 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # If we got to this point everything was successful: move the log file to the experiment directory. mv_vrfy(logfile, EXPTDIR) -def get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS): + +def get_nomads_data( + NOMADS_file_type, + EXPTDIR, + USHdir, + DATE_FIRST_CYCL, + CYCL_HRS, + FCST_LEN_HRS, + LBC_SPEC_INTVL_HRS, +): print("Getting NOMADS online data") print(f"NOMADS_file_type= {NOMADS_file_type}") cd_vrfy(EXPTDIR) NOMADS_script = os.path.join(USHdir, "NOMADS_get_extrn_mdl_files.sh") - run_command(f"""{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ - {date_to_str(DATE_FIRST_CYCL,format="%H")} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}""") + run_command( + f"""{NOMADS_script} \ + {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ + {date_to_str(DATE_FIRST_CYCL,format="%H")} \ + {NOMADS_file_type} \ + {FCST_LEN_HRS} \ + {LBC_SPEC_INTVL_HRS}""" + ) + -def setup_logging(logfile: str = 'log.generate_FV3LAM_wflow') -> None: +def setup_logging(logfile: str = "log.generate_FV3LAM_wflow") -> None: """ Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all messages with detailed timing and routine info in the specified text file. """ - logging.basicConfig(level=logging.DEBUG, - format='%(name)-22s %(levelname)-8s %(message)s', - filename=logfile, - filemode='w') - logging.debug(f'Finished setting up debug file logging in {logfile}') + logging.basicConfig( + level=logging.DEBUG, + format="%(name)-22s %(levelname)-8s %(message)s", + filename=logfile, + filemode="w", + ) + logging.debug(f"Finished setting up debug file logging in {logfile}") console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger().addHandler(console) - logging.debug('Logging set up successfully') + logging.debug("Logging set up successfully") + if __name__ == "__main__": USHdir = os.path.dirname(os.path.abspath(__file__)) - logfile=f'{USHdir}/log.generate_FV3LAM_wflow' + logfile = f"{USHdir}/log.generate_FV3LAM_wflow" # Call the generate_FV3LAM_wflow function defined above to generate the # experiment/workflow. try: generate_FV3LAM_wflow(USHdir, logfile) except: - logging.exception(dedent( - f""" + logging.exception( + dedent( + f""" ********************************************************************* FATAL ERROR: Experiment generation failed. See the error message(s) printed below. @@ -1008,14 +1036,16 @@ def setup_logging(logfile: str = 'log.generate_FV3LAM_wflow') -> None: generation script: {logfile} *********************************************************************\n """ - )) + ) + ) + class Testing(unittest.TestCase): def test_generate_FV3LAM_wflow(self): # run workflows in separate process to avoid conflict between community and nco settings - def run_workflow(USHdir,logfile): - p = Process(target=generate_FV3LAM_wflow,args=(USHdir,logfile)) + def run_workflow(USHdir, logfile): + p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) p.start() p.join() exit_code = p.exitcode @@ -1023,18 +1053,22 @@ def run_workflow(USHdir,logfile): sys.exit(exit_code) USHdir = os.path.dirname(os.path.abspath(__file__)) - logfile='log.generate_FV3LAM_wflow' + logfile = "log.generate_FV3LAM_wflow" SED = get_env_var("SED") # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") - run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") + run_command( + f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" + ) run_workflow(USHdir, logfile) # nco test case set_env_var("OPSROOT", f"{USHdir}/../../nco_dirs") cp_vrfy(f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") - run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") + run_command( + f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" + ) run_workflow(USHdir, logfile) def setUp(self): diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index c6c8d41ee0..98e0c369cd 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -64,22 +64,22 @@ def get_crontab_contents(called_from_cron): __crontab_cmd__ = "/usr/bin/crontab" print_info_msg( - f''' + f""" Getting crontab content with command: ========================================================= {__crontab_cmd__} -l - =========================================================''', + =========================================================""", verbose=DEBUG, ) (_, __crontab_contents__, _) = run_command(f"""{__crontab_cmd__} -l""") print_info_msg( - f''' + f""" Crontab contents: ========================================================= {__crontab_contents__} - =========================================================''', + =========================================================""", verbose=DEBUG, ) @@ -173,11 +173,11 @@ def delete_crontab_line(called_from_cron): # Then record the results back into the user's cron table. # print_info_msg( - f''' + f""" Crontab contents before delete: ========================================================= {crontab_contents} - =========================================================''', + =========================================================""", verbose=True, ) @@ -189,11 +189,11 @@ def delete_crontab_line(called_from_cron): run_command(f"""echo '{crontab_contents}' | {crontab_cmd}""") print_info_msg( - f''' + f""" Crontab contents after delete: ========================================================= {crontab_contents} - =========================================================''', + =========================================================""", verbose=True, ) diff --git a/ush/python_utils/check_for_preexist_dir_file.py b/ush/python_utils/check_for_preexist_dir_file.py index cfd4d50f43..39ad7f1706 100644 --- a/ush/python_utils/check_for_preexist_dir_file.py +++ b/ush/python_utils/check_for_preexist_dir_file.py @@ -7,6 +7,7 @@ from .filesys_cmds_vrfy import rm_vrfy, mv_vrfy from .print_msg import log_info + def check_for_preexist_dir_file(path, method): """Check for a preexisting directory or file and, if present, deal with it according to the specified method @@ -21,10 +22,12 @@ def check_for_preexist_dir_file(path, method): try: check_var_valid_value(method, ["delete", "rename", "quit"]) except ValueError: - errmsg = dedent(f''' - Invalid method for dealing with pre-existing directory specified - method = {method} - ''') + errmsg = dedent( + f""" + Invalid method for dealing with pre-existing directory specified + method = {method} + """ + ) raise ValueError(errmsg) from None if os.path.exists(path): @@ -43,8 +46,10 @@ def check_for_preexist_dir_file(path, method): ) mv_vrfy(path, new_path) else: - raise FileExistsError(dedent( - f""" + raise FileExistsError( + dedent( + f""" Specified directory or file already exists {path}""" - )) + ) + ) diff --git a/ush/python_utils/check_var_valid_value.py b/ush/python_utils/check_var_valid_value.py index a8b88e17a6..0c9bcc49c6 100644 --- a/ush/python_utils/check_var_valid_value.py +++ b/ush/python_utils/check_var_valid_value.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 + def check_var_valid_value(var, values): """Check if specified variable has a valid value diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index c1b69db5ff..1c96d0ba7c 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -60,8 +60,8 @@ def increase_indent(self, flow=False, indentless=False): def str_presenter(dumper, data): if len(data.splitlines()) > 1: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) yaml.add_representer(str, str_presenter) @@ -214,10 +214,13 @@ def load_ini_config(config_file, return_string=0): """Load a config file with a format similar to Microsoft's INI files""" if not os.path.exists(config_file): - raise FileNotFoundError(dedent(f''' - The specified configuration file does not exist: - "{config_file}"''' - )) + raise FileNotFoundError( + dedent( + f''' + The specified configuration file does not exist: + "{config_file}"''' + ) + ) config = configparser.RawConfigParser() config.optionxform = str @@ -233,7 +236,7 @@ def get_ini_value(config, section, key): """Finds the value of a property in a given section""" if not section in config: - raise KeyError(f'Section not found: {section}') + raise KeyError(f"Section not found: {section}") else: return config[section][key] diff --git a/ush/python_utils/print_msg.py b/ush/python_utils/print_msg.py index 8db0f06b03..078d708ba9 100644 --- a/ush/python_utils/print_msg.py +++ b/ush/python_utils/print_msg.py @@ -41,6 +41,7 @@ def print_info_msg(info_msg, verbose=True): return True return False + def log_info(info_msg, verbose=True, dedent_=True): """Function to print information message using the logging module. This function should not be used if python logging has not been initialized. @@ -54,11 +55,10 @@ def log_info(info_msg, verbose=True, dedent_=True): """ # "sys._getframe().f_back.f_code.co_name" returns the name of the calling function - logger=getLogger(sys._getframe().f_back.f_code.co_name) + logger = getLogger(sys._getframe().f_back.f_code.co_name) if verbose: if dedent_: - logger.info(indent(dedent(info_msg), ' ')) + logger.info(indent(dedent(info_msg), " ")) else: logger.info(info_msg) - diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 52f217bb85..2d7bcefa17 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -47,7 +47,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join( os.getcwd(), file_path.lstrip("/") ) + local_file_path = os.path.join(os.getcwd(), file_path.lstrip("/")) if not os.path.exists(local_file_path): logging.info(f"File does not exist: {local_file_path}") unavailable["hpss"] = source_paths @@ -493,7 +493,6 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) if isinstance(archive_internal_dirs, dict): archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, [""]) - # which_archive matters for choosing the correct file names within, # but we can safely just try all options for the # archive_internal_dir diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index e285fc92ed..80baba6eb0 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -185,7 +185,9 @@ def setUp(self): ), ) - cd_vrfy(f'{EXPTDIR}{os.sep}{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem2') + cd_vrfy( + f'{EXPTDIR}{os.sep}{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem2' + ) set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index e713ebc2e0..59c3b10401 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -27,6 +27,7 @@ from set_namelist import set_namelist + def set_FV3nml_sfc_climo_filenames(): """ This function sets the values of the variables in @@ -47,7 +48,7 @@ def set_FV3nml_sfc_climo_filenames(): import_vars() # fixed file mapping variables - fixed_cfg = load_config_file(os.path.join(PARMdir,"fixed_files_mapping.yaml")) + fixed_cfg = load_config_file(os.path.join(PARMdir, "fixed_files_mapping.yaml")) IMPORTS = ["SFC_CLIMO_FIELDS", "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"] import_vars(dictionary=flatten_dict(fixed_cfg), env_vars=IMPORTS) @@ -171,4 +172,3 @@ def setUp(self): set_env_var("CRES", "C3357") set_env_var("RUN_ENVIR", "nco") set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) - diff --git a/ush/set_cycle_dates.py b/ush/set_cycle_dates.py index 09adf55e88..069a3891b1 100644 --- a/ush/set_cycle_dates.py +++ b/ush/set_cycle_dates.py @@ -43,6 +43,13 @@ def test_set_cycle_dates(self): incr_cycl_freq=6, ) self.assertEqual( - cdates, ["2022010106", "2022010112", "2022010118", - "2022010200", "2022010206", "2022010212"] + cdates, + [ + "2022010106", + "2022010112", + "2022010118", + "2022010200", + "2022010206", + "2022010212", + ], ) diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py index ce8cd02ffe..dc2269b2ce 100644 --- a/ush/set_gridparams_ESGgrid.py +++ b/ush/set_gridparams_ESGgrid.py @@ -5,11 +5,11 @@ from datetime import datetime, timedelta from python_utils import ( - import_vars, - set_env_var, - print_input_args, - load_config_file, - flatten_dict, + import_vars, + set_env_var, + print_input_args, + load_config_file, + flatten_dict, ) @@ -35,7 +35,7 @@ def set_gridparams_ESGgrid(lon_ctr, lat_ctr, nx, ny, halo_width, delx, dely, paz # get constants IMPORTS = ["RADIUS_EARTH", "DEGS_PER_RADIAN"] USHdir = os.path.dirname(os.path.abspath(__file__)) - constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + constants_cfg = load_config_file(os.path.join(USHdir, "constants.yaml")) import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) # diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py index 47253a1c73..70a309c3fd 100644 --- a/ush/set_gridparams_GFDLgrid.py +++ b/ush/set_gridparams_GFDLgrid.py @@ -63,7 +63,7 @@ def set_gridparams_GFDLgrid( import_vars(env_vars=IMPORTS) IMPORTS = ["NH4"] USHdir = os.path.dirname(os.path.abspath(__file__)) - constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + constants_cfg = load_config_file(os.path.join(USHdir, "constants.yaml")) import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) # diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index 23c3e985e5..f8536948aa 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -18,7 +18,12 @@ ) -def set_ozone_param(ccpp_phys_suite_fp, CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, FIXgsm_FILES_TO_COPY_TO_FIXam, VERBOSE): +def set_ozone_param( + ccpp_phys_suite_fp, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE, +): """Function that does the following: (1) Determines the ozone parameterization being used by checking in the CCPP physics suite XML. @@ -90,8 +95,10 @@ def set_ozone_param(ccpp_phys_suite_fp, CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, F fixgsm_ozone_fn = "global_o3prdlos.f77" ozone_param = "ozphys" else: - raise KeyError(f'Unknown or no ozone parameterization specified in the ' - 'CCPP physics suite file "{ccpp_phys_suite_fp}"') + raise KeyError( + f"Unknown or no ozone parameterization specified in the " + 'CCPP physics suite file "{ccpp_phys_suite_fp}"' + ) # # ----------------------------------------------------------------------- # @@ -151,10 +158,16 @@ def set_ozone_param(ccpp_phys_suite_fp, CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, F CCPP suite definition file), the array specifying the mapping between the symlinks that need to be created in the cycle directories and the files in the FIXam directory is: - """, verbose=VERBOSE) - log_info(f""" + """, + verbose=VERBOSE, + ) + log_info( + f""" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} - """, verbose=VERBOSE, dedent_=False) + """, + verbose=VERBOSE, + dedent_=False, + ) else: @@ -178,7 +191,7 @@ def test_set_ozone_param(self): f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, FIXgsm_FILES_TO_COPY_TO_FIXam, - VERBOSE=True + VERBOSE=True, ), ) diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index 6b432b8f03..8cf9382947 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -41,12 +41,13 @@ def set_predef_grid_params(): try: params_dict = params_dict[PREDEF_GRID_NAME] except KeyError: - errmsg = dedent(f''' + errmsg = dedent( + f""" PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml - Check your config file settings.''') + Check your config file settings.""" + ) raise Exception(errmsg) from None - # if QUILTING = False, remove key if not QUILTING: params_dict.pop("QUILTING") diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 40eb00bd9b..a70bfe19f9 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -17,8 +17,12 @@ ) -def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn, - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, FIXgsm_FILES_TO_COPY_TO_FIXam): +def set_thompson_mp_fix_files( + ccpp_phys_suite_fp, + thompson_mp_climo_fn, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, +): """Function that first checks whether the Thompson microphysics parameterization is being called by the selected physics suite. If not, it sets the output variable whose name is specified by @@ -41,7 +45,7 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn, print_input_args(locals()) # import some environment variables - IMPORTS=["EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", "CCPP_PHYS_SUITE"] + IMPORTS = ["EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", "CCPP_PHYS_SUITE"] import_vars(env_vars=IMPORTS) # @@ -129,7 +133,7 @@ def test_set_thompson_mp_fix_files(self): f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", "Thompson_MP_MONTHLY_CLIMO.nc", CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - FIXgsm_FILES_TO_COPY_TO_FIXam, + FIXgsm_FILES_TO_COPY_TO_FIXam, ), ) @@ -195,4 +199,3 @@ def setUp(self): "HGT.Beljaars_filtered.lat-lon.30s_res.nc", "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] - diff --git a/ush/setup.py b/ush/setup.py index 9a8569f570..78bcafa7c0 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -82,42 +82,64 @@ def setup(): # EXPT_DEFAULT_CONFIG_FN = "config_defaults.yaml" cfg_d = load_config_file(os.path.join(USHdir, EXPT_DEFAULT_CONFIG_FN)) - import_vars(dictionary=flatten_dict(cfg_d), - env_vars=["EXPT_CONFIG_FN", - "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", - "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) - + import_vars( + dictionary=flatten_dict(cfg_d), + env_vars=[ + "EXPT_CONFIG_FN", + "EXTRN_MDL_NAME_ICS", + "EXTRN_MDL_NAME_LBCS", + "FV3GFS_FILE_FMT_ICS", + "FV3GFS_FILE_FMT_LBCS", + ], + ) # Load the user config file, then ensure all user-specified - # variables correspond to a default value. + # variables correspond to a default value. if not os.path.exists(EXPT_CONFIG_FN): - raise FileNotFoundError(f'User config file not found: EXPT_CONFIG_FN = {EXPT_CONFIG_FN}') + raise FileNotFoundError( + f"User config file not found: EXPT_CONFIG_FN = {EXPT_CONFIG_FN}" + ) try: cfg_u = load_config_file(os.path.join(USHdir, EXPT_CONFIG_FN)) except: - errmsg = dedent(f'''\n - Could not load YAML config file: {EXPT_CONFIG_FN} - Reference the above traceback for more information. - ''') + errmsg = dedent( + f"""\n + Could not load YAML config file: {EXPT_CONFIG_FN} + Reference the above traceback for more information. + """ + ) raise Exception(errmsg) cfg_u = flatten_dict(cfg_u) for key in cfg_u: if key not in flatten_dict(cfg_d): - raise Exception(dedent(f''' - User-specified variable "{key}" in {EXPT_CONFIG_FN} is not valid. - Check {EXPT_DEFAULT_CONFIG_FN} for allowed user-specified variables.\n''')) + raise Exception( + dedent( + f""" + User-specified variable "{key}" in {EXPT_CONFIG_FN} is not valid. + Check {EXPT_DEFAULT_CONFIG_FN} for allowed user-specified variables.\n""" + ) + ) # Mandatory variables *must* be set in the user's config; the default value is invalid - mandatory = ['MACHINE'] + mandatory = ["MACHINE"] for val in mandatory: if val not in cfg_u: - raise Exception(f'Mandatory variable "{val}" not found in user config file {EXPT_CONFIG_FN}') + raise Exception( + f'Mandatory variable "{val}" not found in user config file {EXPT_CONFIG_FN}' + ) - import_vars(dictionary=cfg_u, env_vars=["MACHINE", - "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", - "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) + import_vars( + dictionary=cfg_u, + env_vars=[ + "MACHINE", + "EXTRN_MDL_NAME_ICS", + "EXTRN_MDL_NAME_LBCS", + "FV3GFS_FILE_FMT_ICS", + "FV3GFS_FILE_FMT_LBCS", + ], + ) # # ----------------------------------------------------------------------- # @@ -130,23 +152,25 @@ def setup(): global MACHINE, EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS MACHINE_FILE = os.path.join(USHdir, "machine", f"{lowercase(MACHINE)}.yaml") if not os.path.exists(MACHINE_FILE): - raise FileNotFoundError(dedent( - f""" + raise FileNotFoundError( + dedent( + f""" The machine file {MACHINE_FILE} does not exist. Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" - )) + ) + ) machine_cfg = load_config_file(MACHINE_FILE) # ics and lbcs - def get_location(xcs,fmt): - if ("data" in machine_cfg) and (xcs in machine_cfg["data"]): - v = machine_cfg["data"][xcs] - if not isinstance(v,dict): - return v - else: - return v[fmt] - else: - return "" + def get_location(xcs, fmt): + if ("data" in machine_cfg) and (xcs in machine_cfg["data"]): + v = machine_cfg["data"][xcs] + if not isinstance(v, dict): + return v + else: + return v[fmt] + else: + return "" EXTRN_MDL_SYSBASEDIR_ICS = get_location(EXTRN_MDL_NAME_ICS, FV3GFS_FILE_FMT_ICS) EXTRN_MDL_SYSBASEDIR_LBCS = get_location(EXTRN_MDL_NAME_LBCS, FV3GFS_FILE_FMT_LBCS) @@ -154,10 +178,12 @@ def get_location(xcs,fmt): # remove the data key and provide machine specific default values for cfg_d if "data" in machine_cfg: machine_cfg.pop("data") - machine_cfg.update({ - "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, - "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, - }) + machine_cfg.update( + { + "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, + "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, + } + ) machine_cfg = flatten_dict(machine_cfg) update_dict(machine_cfg, cfg_d) @@ -180,7 +206,9 @@ def get_location(xcs,fmt): MACHINE = uppercase(MACHINE) # Load fixed-files mapping file - cfg_f = load_config_file(os.path.join(USHdir, os.pardir, "parm", "fixed_files_mapping.yaml")) + cfg_f = load_config_file( + os.path.join(USHdir, os.pardir, "parm", "fixed_files_mapping.yaml") + ) import_vars(dictionary=flatten_dict(cfg_f)) # Load constants file and save its contents to a variable for later @@ -307,7 +335,7 @@ def get_location(xcs,fmt): or (len(ISEED_SPP) != N_VAR_SPP) ): raise Exception( - f''' + f""" All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist variables set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: @@ -319,7 +347,7 @@ def get_location(xcs,fmt): SPP_SIGTOP2 (length {len(SPP_SIGTOP2)}) SPP_STDDEV_CUTOFF (length {len(SPP_STDDEV_CUTOFF)}) ISEED_SPP (length {len(ISEED_SPP)}) - ''' + """ ) # # ----------------------------------------------------------------------- @@ -337,7 +365,7 @@ def get_location(xcs,fmt): or (len(LSM_SPP_TSCALE) != N_VAR_LNDP) ): raise Exception( - f''' + f""" All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: @@ -345,16 +373,14 @@ def get_location(xcs,fmt): LSM_SPP_MAG_LIST (length {len(LSM_SPP_MAG_LIST)}) LSM_SPP_LSCALE (length {len(LSM_SPP_LSCALE)}) LSM_SPP_TSCALE (length {len(LSM_SPP_TSCALE)}) - ''' + """ ) # # The current script should be located in the ush subdirectory of the # workflow directory. Thus, the workflow directory is the one above the # directory of the current script. # - HOMEdir = os.path.abspath( - os.path.dirname(__file__) + os.sep + os.pardir - ) + HOMEdir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir) # # ----------------------------------------------------------------------- @@ -385,22 +411,25 @@ def get_location(xcs,fmt): try: UFS_WTHR_MDL_DIR = get_ini_value(cfg, external_name, property_name) except KeyError: - errmsg = dedent(f''' - Externals configuration file {mng_extrns_cfg_fn} - does not contain "{external_name}".''') + errmsg = dedent( + f""" + Externals configuration file {mng_extrns_cfg_fn} + does not contain "{external_name}".""" + ) raise Exception(errmsg) from None - UFS_WTHR_MDL_DIR = os.path.join(HOMEdir, UFS_WTHR_MDL_DIR) if not os.path.exists(UFS_WTHR_MDL_DIR): - raise FileNotFoundError(dedent( - f""" + raise FileNotFoundError( + dedent( + f""" The base directory in which the FV3 source code should be located (UFS_WTHR_MDL_DIR) does not exist: UFS_WTHR_MDL_DIR = \"{UFS_WTHR_MDL_DIR}\" Please clone the external repository containing the code in this directory, build the executable, and then rerun the workflow.""" - )) + ) + ) # # Define some other useful paths # @@ -432,17 +461,28 @@ def get_location(xcs,fmt): RELATIVE_LINK_FLAG = "--relative" # Mandatory variables *must* be set in the user's config or the machine file; the default value is invalid - mandatory = ['NCORES_PER_NODE', 'FIXgsm', 'FIXaer', 'FIXlut', 'TOPO_DIR', 'SFC_CLIMO_INPUT_DIR'] + mandatory = [ + "NCORES_PER_NODE", + "FIXgsm", + "FIXaer", + "FIXlut", + "TOPO_DIR", + "SFC_CLIMO_INPUT_DIR", + ] globalvars = globals() for val in mandatory: # globals() returns dictionary of global variables if not globalvars[val]: - raise Exception(dedent(f''' - Mandatory variable "{val}" not found in: - user config file {EXPT_CONFIG_FN} - OR - machine file {MACHINE_FILE} - ''')) + raise Exception( + dedent( + f""" + Mandatory variable "{val}" not found in: + user config file {EXPT_CONFIG_FN} + OR + machine file {MACHINE_FILE} + """ + ) + ) # # ----------------------------------------------------------------------- @@ -482,10 +522,13 @@ def get_location(xcs,fmt): # if WORKFLOW_MANAGER is not None: if not ACCOUNT: - raise Exception(dedent(f''' + raise Exception( + dedent( + f""" ACCOUNT must be specified in config or machine file if using a workflow manager. - WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n''' - )) + WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n""" + ) + ) # # ----------------------------------------------------------------------- # @@ -527,50 +570,60 @@ def get_location(xcs,fmt): # Make sure RESTART_INTERVAL is set to an integer value if not isinstance(RESTART_INTERVAL, int): - raise Exception(f"\nRESTART_INTERVAL = {RESTART_INTERVAL}, must be an integer value\n") + raise Exception( + f"\nRESTART_INTERVAL = {RESTART_INTERVAL}, must be an integer value\n" + ) # Check that input dates are in a date format # get dictionary of all variables allvars = dict(globals()) allvars.update(locals()) - dates = ['DATE_FIRST_CYCL', 'DATE_LAST_CYCL'] + dates = ["DATE_FIRST_CYCL", "DATE_LAST_CYCL"] for val in dates: if not isinstance(allvars[val], datetime.date): - raise Exception(dedent(f''' - Date variable {val}={allvars[val]} is not in a valid date format + raise Exception( + dedent( + f""" + Date variable {val}={allvars[val]} is not in a valid date format - For examples of valid formats, see the users guide. - ''')) + For examples of valid formats, see the users guide. + """ + ) + ) # If using a custom post configuration file, make sure that it exists. if USE_CUSTOM_POST_CONFIG_FILE: try: - #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + # os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch if not os.path.exists(CUSTOM_POST_CONFIG_FP): raise except: - raise FileNotFoundError(dedent( - f''' + raise FileNotFoundError( + dedent( + f""" USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} - could not be found.''' - )) from None + could not be found.""" + ) + ) from None # If using external CRTM fix files to allow post-processing of synthetic # satellite products from the UPP, make sure the CRTM fix file directory exists. if USE_CRTM: try: - #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + # os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch if not os.path.exists(CRTM_DIR): raise except: - raise FileNotFoundError(dedent( - f''' + raise FileNotFoundError( + dedent( + f""" USE_CRTM has been set, but the external CRTM fix file directory: CRTM_DIR = {CRTM_DIR} - could not be found.''' - )) from None + could not be found.""" + ) + ) from None # The forecast length (in integer hours) cannot contain more than 3 characters. # Thus, its maximum value is 999. @@ -586,7 +639,7 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible - # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an + # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an # array of forecast hours at which the boundary values will be updated. # # ----------------------------------------------------------------------- @@ -632,11 +685,7 @@ def get_location(xcs,fmt): # get dictionary of all variables allvars = dict(globals()) allvars.update(locals()) - vlist = ['DT_ATMOS', - 'LAYOUT_X', - 'LAYOUT_Y', - 'BLOCKSIZE', - 'EXPT_SUBDIR'] + vlist = ["DT_ATMOS", "LAYOUT_X", "LAYOUT_Y", "BLOCKSIZE", "EXPT_SUBDIR"] for val in vlist: if not allvars[val]: raise Exception(f"\nMandatory variable '{val}' has not been set\n") @@ -745,21 +794,25 @@ def get_location(xcs,fmt): try: check_for_preexist_dir_file(EXPTDIR, PREEXISTING_DIR_METHOD) except ValueError: - logger.exception(f''' - Check that the following values are valid: - EXPTDIR {EXPTDIR} - PREEXISTING_DIR_METHOD {PREEXISTING_DIR_METHOD} - ''') + logger.exception( + f""" + Check that the following values are valid: + EXPTDIR {EXPTDIR} + PREEXISTING_DIR_METHOD {PREEXISTING_DIR_METHOD} + """ + ) raise except FileExistsError: - errmsg = dedent(f''' - EXPTDIR ({EXPTDIR}) already exists, and PREEXISTING_DIR_METHOD = {PREEXISTING_DIR_METHOD} - - To ignore this error, delete the directory, or set - PREEXISTING_DIR_METHOD = delete, or - PREEXISTING_DIR_METHOD = rename - in your config file. - ''') + errmsg = dedent( + f""" + EXPTDIR ({EXPTDIR}) already exists, and PREEXISTING_DIR_METHOD = {PREEXISTING_DIR_METHOD} + + To ignore this error, delete the directory, or set + PREEXISTING_DIR_METHOD = delete, or + PREEXISTING_DIR_METHOD = rename + in your config file. + """ + ) raise FileExistsError(errmsg) from None # # ----------------------------------------------------------------------- @@ -801,22 +854,35 @@ def get_location(xcs,fmt): # Main directory locations if RUN_ENVIR == "nco": - try: OPSROOT = os.path.abspath(f"{EXPT_BASEDIR}{os.sep}..{os.sep}nco_dirs") \ - if OPSROOT is None else OPSROOT - except NameError: OPSROOT = EXPTDIR - try: COMROOT - except NameError: COMROOT = os.path.join(OPSROOT, "com") - try: PACKAGEROOT - except NameError: PACKAGEROOT = os.path.join(OPSROOT, "packages") - try: DATAROOT - except NameError: DATAROOT = os.path.join(OPSROOT, "tmp") - try: DCOMROOT - except NameError: DCOMROOT = os.path.join(OPSROOT, "dcom") + try: + OPSROOT = ( + os.path.abspath(f"{EXPT_BASEDIR}{os.sep}..{os.sep}nco_dirs") + if OPSROOT is None + else OPSROOT + ) + except NameError: + OPSROOT = EXPTDIR + try: + COMROOT + except NameError: + COMROOT = os.path.join(OPSROOT, "com") + try: + PACKAGEROOT + except NameError: + PACKAGEROOT = os.path.join(OPSROOT, "packages") + try: + DATAROOT + except NameError: + DATAROOT = os.path.join(OPSROOT, "tmp") + try: + DCOMROOT + except NameError: + DCOMROOT = os.path.join(OPSROOT, "dcom") COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - LOGDIR = os.path.join(OPSROOT,"output") + LOGDIR = os.path.join(OPSROOT, "output") else: @@ -830,24 +896,42 @@ def get_location(xcs,fmt): LOGDIR = os.path.join(EXPTDIR, "log") - try: DBNROOT - except NameError: DBNROOT = None - try: SENDECF - except NameError: SENDECF = False - try: SENDDBN - except NameError: SENDDBN = False - try: SENDDBN_NTC - except NameError: SENDDBN_NTC = False - try: SENDCOM - except NameError: SENDCOM = False - try: SENDWEB - except NameError: SENDWEB = False - try: KEEPDATA - except NameError: KEEPDATA = True - try: MAILTO - except NameError: MAILTO = None - try: MAILCC - except NameError: MAILCC = None + try: + DBNROOT + except NameError: + DBNROOT = None + try: + SENDECF + except NameError: + SENDECF = False + try: + SENDDBN + except NameError: + SENDDBN = False + try: + SENDDBN_NTC + except NameError: + SENDDBN_NTC = False + try: + SENDCOM + except NameError: + SENDCOM = False + try: + SENDWEB + except NameError: + SENDWEB = False + try: + KEEPDATA + except NameError: + KEEPDATA = True + try: + MAILTO + except NameError: + MAILTO = None + try: + MAILCC + except NameError: + MAILCC = None # create NCO directories if RUN_ENVIR == "nco": @@ -873,7 +957,7 @@ def get_location(xcs,fmt): POST_OUTPUT_DOMAIN_NAME = POST_OUTPUT_DOMAIN_NAME or PREDEF_GRID_NAME if type(POST_OUTPUT_DOMAIN_NAME) != int: - POST_OUTPUT_DOMAIN_NAME = lowercase(POST_OUTPUT_DOMAIN_NAME) + POST_OUTPUT_DOMAIN_NAME = lowercase(POST_OUTPUT_DOMAIN_NAME) if POST_OUTPUT_DOMAIN_NAME is None: if PREDEF_GRID_NAME is None: @@ -1028,11 +1112,11 @@ def get_location(xcs,fmt): # OZONE_PARAM = set_ozone_param( - CCPP_PHYS_SUITE_IN_CCPP_FP, - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - FIXgsm_FILES_TO_COPY_TO_FIXam, - VERBOSE=VERBOSE, - ) + CCPP_PHYS_SUITE_IN_CCPP_FP, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE=VERBOSE, + ) # # ----------------------------------------------------------------------- @@ -1224,16 +1308,18 @@ def get_location(xcs,fmt): # experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_GRID: - if (GRID_DIR is None): + if GRID_DIR is None: GRID_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" + msg = dedent( + f""" GRID_DIR not specified! Setting GRID_DIR = {GRID_DIR} - """) + """ + ) logger.warning(msg) - if not os.path.exists(GRID_DIR): + if not os.path.exists(GRID_DIR): raise FileNotFoundError( f''' The directory (GRID_DIR) that should contain the pregenerated grid files @@ -1249,13 +1335,15 @@ def get_location(xcs,fmt): # the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_OROG: - if (OROG_DIR is None): + if OROG_DIR is None: OROG_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" + msg = dedent( + f""" OROG_DIR not specified! Setting OROG_DIR = {OROG_DIR} - """) + """ + ) logger.warning(msg) if not os.path.exists(OROG_DIR): @@ -1274,13 +1362,15 @@ def get_location(xcs,fmt): # a predefined location under the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_SFC_CLIMO: - if (SFC_CLIMO_DIR is None): + if SFC_CLIMO_DIR is None: SFC_CLIMO_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" + msg = dedent( + f""" SFC_CLIMO_DIR not specified! Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} - """) + """ + ) logger.warning(msg) if not os.path.exists(SFC_CLIMO_DIR): @@ -1366,7 +1456,6 @@ def get_location(xcs,fmt): "STRETCH_FAC": STRETCH_FAC, } - # Extract the basic grid params from the dictionary (LON_CTR, LAT_CTR, NX, NY, NHW, STRETCH_FAC) = ( grid_params[k] for k in ["LON_CTR", "LAT_CTR", "NX", "NY", "NHW", "STRETCH_FAC"] @@ -1375,7 +1464,7 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Create a new experiment directory. For platforms with no workflow + # Create a new experiment directory. For platforms with no workflow # manager we need to create LOGDIR as well, since it won't be created # later at runtime. # @@ -1486,10 +1575,14 @@ def get_location(xcs,fmt): if WRITE_DOPOST: # Turn off run_post if RUN_TASK_RUN_POST: - logger.warning(dedent(f""" - Inline post is turned on, deactivating post-processing tasks: - RUN_TASK_RUN_POST = False - """)) + logger.warning( + dedent( + f""" + Inline post is turned on, deactivating post-processing tasks: + RUN_TASK_RUN_POST = False + """ + ) + ) RUN_TASK_RUN_POST = False # Check if SUB_HOURLY_POST is on @@ -1514,12 +1607,12 @@ def get_location(xcs,fmt): if VERBOSE: log_info( - f""" + f""" The number of MPI tasks for the forecast (including those for the write component if it is being used) are: PE_MEMBER01 = {PE_MEMBER01}""", - verbose=VERBOSE, - ) + verbose=VERBOSE, + ) # # ----------------------------------------------------------------------- # diff --git a/ush/test_retrieve_data.py b/ush/test_retrieve_data.py index 3ffdf4744c..dc7a775f05 100644 --- a/ush/test_retrieve_data.py +++ b/ush/test_retrieve_data.py @@ -1,4 +1,4 @@ -''' +""" Functional test suite for gathering data using retreve_data.py. The tests reflect some use cases of gathering various model input data @@ -14,7 +14,7 @@ To run a single test: python -m unittest -b test_retrieve_data.FunctionalTesting.test_rap_lbcs_from_aws -''' +""" import glob import os import tempfile @@ -22,22 +22,21 @@ import retrieve_data + class FunctionalTesting(unittest.TestCase): - ''' Test class for retrieve data ''' + """Test class for retrieve data""" def setUp(self): self.path = os.path.dirname(__file__) - self.config = f'{self.path}/../parm/data_locations.yml' + self.config = f"{self.path}/../parm/data_locations.yml" - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_fv3gfs_grib2_lbcs_from_hpss(self): - ''' Get FV3GFS grib2 files from HPSS for LBCS, offset by 6 hours - - ''' + """Get FV3GFS grib2 files from HPSS for LBCS, offset by 6 hours""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -56,18 +55,18 @@ def test_fv3gfs_grib2_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 3) - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_fv3gfs_netcdf_lbcs_from_hpss(self): - ''' Get FV3GFS netcdf files from HPSS for LBCS. Tests fcst lead + """Get FV3GFS netcdf files from HPSS for LBCS. Tests fcst lead times > 40 hours, since they come from a different archive file. - ''' + """ - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -86,20 +85,19 @@ def test_fv3gfs_netcdf_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 2) # GDAS Tests def test_gdas_ics_from_aws(self): - ''' In real time, GDAS is used for LBCS with a 6 hour offset. - ''' + """In real time, GDAS is used for LBCS with a 6 hour offset.""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = f'mem{{mem:03d}}' + out_path_tmpl = f"mem{{mem:03d}}" args = [ '--anl_or_fcst', 'anl', @@ -120,22 +118,19 @@ def test_gdas_ics_from_aws(self): for mem in [9, 10]: files_on_disk = glob.glob( - os.path.join(out_path_tmpl.format(mem=mem), '*') - ) + os.path.join(out_path_tmpl.format(mem=mem), "*") + ) self.assertEqual(len(files_on_disk), 2) - # GEFS Tests def test_gefs_grib2_ics_from_aws(self): - ''' Get GEFS grib2 a & b files for ICS offset by 6 hours. + """Get GEFS grib2 a & b files for ICS offset by 6 hours.""" - ''' - - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = f'mem{{mem:03d}}' + out_path_tmpl = f"mem{{mem:03d}}" args = [ '--anl_or_fcst', 'anl', @@ -155,19 +150,17 @@ def test_gefs_grib2_ics_from_aws(self): # Verify files exist in temp dir for mem in [1, 2]: files_on_disk = glob.glob( - os.path.join(out_path_tmpl.format(mem=mem), '*') - ) + os.path.join(out_path_tmpl.format(mem=mem), "*") + ) self.assertEqual(len(files_on_disk), 2) - - # HRRR Tests - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_hrrr_ics_from_hpss(self): - ''' Get HRRR ICS from hpss ''' + """Get HRRR ICS from hpss""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -185,16 +178,16 @@ def test_hrrr_ics_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_hrrr_lbcs_from_hpss(self): - ''' Get HRRR LBCS from hpss for 3 hour boundary conditions ''' + """Get HRRR LBCS from hpss for 3 hour boundary conditions""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -212,15 +205,15 @@ def test_hrrr_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) def test_hrrr_ics_from_aws(self): - ''' Get HRRR ICS from aws ''' + """Get HRRR ICS from aws""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -238,15 +231,15 @@ def test_hrrr_ics_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) def test_hrrr_lbcs_from_aws(self): - ''' Get HRRR LBCS from aws for 3 hour boundary conditions ''' + """Get HRRR LBCS from aws for 3 hour boundary conditions""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -264,16 +257,16 @@ def test_hrrr_lbcs_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) # RAP tests def test_rap_ics_from_aws(self): - ''' Get RAP ICS from aws offset by 3 hours ''' + """Get RAP ICS from aws offset by 3 hours""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -291,16 +284,16 @@ def test_rap_ics_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) def test_rap_lbcs_from_aws(self): - ''' Get RAP LBCS from aws for 6 hour boundary conditions offset - by 3 hours. Use 09Z start time for longer LBCS.''' + """Get RAP LBCS from aws for 6 hour boundary conditions offset + by 3 hours. Use 09Z start time for longer LBCS.""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -318,6 +311,6 @@ def test_rap_lbcs_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 5) From 5abf5305e962dd664bad432ea7b06caf24417d43 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 22 Oct 2022 21:42:21 +0000 Subject: [PATCH 6/9] Add pipefail in strict error checking mode. --- ush/preamble.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/preamble.sh b/ush/preamble.sh index 0357ace76f..4c75665ed6 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -91,7 +91,7 @@ TRACE=${DEBUG:-"FALSE"} if [[ $STRICT == "TRUE" ]]; then # Exit on error and undefined variable - set -eu + set -euo pipefail fi if [[ $TRACE == "TRUE" ]]; then # Turn on debugging From 093b146601f028b7a1ed96e576c1f8f3f84372aa Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 25 Oct 2022 15:29:18 +0000 Subject: [PATCH 7/9] Remove debugging comments from UFS_plot --- ush/UFS_plot_domains.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/UFS_plot_domains.py b/ush/UFS_plot_domains.py index 9068fa4163..847cd96725 100755 --- a/ush/UFS_plot_domains.py +++ b/ush/UFS_plot_domains.py @@ -128,7 +128,6 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): - # print("Hello from a function") # This function takes the lambert domain we have defined, lambert_map, as well as # pps (the number of points to interpolate and draw for each side of the lambert "rectangle"), @@ -187,9 +186,6 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): # Need to replace final instruction with Path.CLOSEPOLY instructions[-1] = Path.CLOSEPOLY - # print("vertices=", vertices) - # print("instructions=", instructions) - return vertices, instructions From 2ed27333158bc2315fa6b1d6860c613a9c40ef84 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 25 Oct 2022 17:01:39 +0000 Subject: [PATCH 8/9] More formatting changes. --- ush/check_python_version.py | 22 +-- ush/create_diag_table_file.py | 33 ++-- ush/create_model_configure_file.py | 24 +-- ush/generate_FV3LAM_wflow.py | 110 ++++++------- ush/get_crontab_contents.py | 14 +- ush/link_fix.py | 6 +- ush/python_utils/config_parser.py | 4 +- ush/python_utils/create_symlink_to_file.py | 16 +- ush/python_utils/filesys_cmds_vrfy.py | 2 +- ush/python_utils/get_charvar_from_netcdf.py | 14 +- ush/python_utils/print_input_args.py | 8 +- ush/set_FV3nml_ens_stoch_seeds.py | 28 ++-- ush/set_FV3nml_sfc_climo_filenames.py | 24 +-- ush/set_ozone_param.py | 6 +- ush/set_predef_grid_params.py | 4 +- ush/set_thompson_mp_fix_files.py | 2 +- ush/setup.py | 166 ++++++++++---------- 17 files changed, 241 insertions(+), 242 deletions(-) diff --git a/ush/check_python_version.py b/ush/check_python_version.py index 3888efdef2..afc3dac62f 100755 --- a/ush/check_python_version.py +++ b/ush/check_python_version.py @@ -19,8 +19,8 @@ def check_python_version(): logging.error( dedent( """ - Error: Missing python package required by the SRW app - """ + Error: Missing python package required by the SRW app + """ ) ) raise @@ -31,8 +31,8 @@ def check_python_version(): logging.error( dedent( f""" - Error: python version must be 3.6 or higher - Your python version is: {major}.{minor}""" + Error: python version must be 3.6 or higher + Your python version is: {major}.{minor}""" ) ) raise Exception("Python version below 3.6") @@ -45,13 +45,13 @@ def check_python_version(): logging.exception( dedent( f""" - ************************************************************************* - FATAL ERROR: - The system does not meet minimum requirements for running the SRW app. - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - *************************************************************************\n - """ + ************************************************************************* + FATAL ERROR: + The system does not meet minimum requirements for running the SRW app. + Instructions for setting up python environments can be found on the web: + https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started + *************************************************************************\n + """ ) ) sys.exit(1) diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index c939c6ee6b..15a42729f4 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -36,26 +36,25 @@ def create_diag_table_file(run_dir): # create a diagnostic table file within the specified run directory print_info_msg( - f''' - Creating a diagnostics table file (\"{DIAG_TABLE_FN}\") in the specified + f""" + Creating a diagnostics table file ('{DIAG_TABLE_FN}') in the specified run directory... - run_dir = \"{run_dir}\"''', + run_dir = '{run_dir}'""", verbose=VERBOSE, ) diag_table_fp = os.path.join(run_dir, DIAG_TABLE_FN) print_info_msg( - f''' - + f""" Using the template diagnostics table file: diag_table_tmpl_fp = {DIAG_TABLE_TMPL_FP} to create: - diag_table_fp = \"{diag_table_fp}\"''', + diag_table_fp = '{diag_table_fp}'""", verbose=VERBOSE, ) @@ -65,9 +64,9 @@ def create_diag_table_file(run_dir): print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{DIAG_TABLE_FN}\" - file has been set as follows:\n - settings =\n\n""" + The variable 'settings' specifying values to be used in the '{DIAG_TABLE_FN}' + file has been set as follows:\n + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -82,14 +81,14 @@ def create_diag_table_file(run_dir): print_err_msg_exit( dedent( f""" - Call to python script fill_jinja_template.py to create a \"{DIAG_TABLE_FN}\" - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template diag table file: - DIAG_TABLE_TMPL_FP = \"{DIAG_TABLE_TMPL_FP}\" - Full path to output diag table file: - diag_table_fp = \"{diag_table_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a '{DIAG_TABLE_FN}' + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template diag table file: + DIAG_TABLE_TMPL_FP = '{DIAG_TABLE_TMPL_FP}' + Full path to output diag table file: + diag_table_fp = '{diag_table_fp}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 1cd445a3e9..7015e59729 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -52,10 +52,10 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # print_info_msg( - f''' - Creating a model configuration file (\"{MODEL_CONFIG_FN}\") in the specified + f""" + Creating a model configuration file ('{MODEL_CONFIG_FN}') in the specified run directory (run_dir): - run_dir = \"{run_dir}\"''', + run_dir = '{run_dir}'""", verbose=VERBOSE, ) # @@ -190,7 +190,7 @@ def create_model_configure_file( print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{MODEL_CONFIG_FN}\" + The variable 'settings' specifying values to be used in the '{MODEL_CONFIG_FN}' file has been set as follows:\n settings =\n\n""" ) @@ -223,14 +223,14 @@ def create_model_configure_file( print_err_msg_exit( dedent( f""" - Call to python script fill_jinja_template.py to create a \"{MODEL_CONFIG_FN}\" - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template model config file: - MODEL_CONFIG_TMPL_FP = \"{MODEL_CONFIG_TMPL_FP}\" - Full path to output model config file: - model_config_fp = \"{model_config_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a '{MODEL_CONFIG_FN}' + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template model config file: + MODEL_CONFIG_TMPL_FP = '{MODEL_CONFIG_TMPL_FP}' + Full path to output model config file: + model_config_fp = '{model_config_fp}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index e5c26703d3..969485fce0 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -101,11 +101,11 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> template_xml_fp = os.path.join(PARMdir, WFLOW_XML_FN) log_info( - f''' + f""" Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML file (template_xml_fp): - template_xml_fp = \"{template_xml_fp}\" - WFLOW_XML_FP = \"{WFLOW_XML_FP}\"''' + template_xml_fp = '{template_xml_fp}' + WFLOW_XML_FP = '{WFLOW_XML_FP}'""" ) ensmem_indx_name = "" @@ -397,7 +397,7 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> log_info( f""" - The variable \"settings\" specifying values of the rococo XML variables + The variable 'settings' specifying values of the rococo XML variables has been set as follows: #----------------------------------------------------------------------- settings =\n\n""", @@ -417,15 +417,15 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> logging.exception( dedent( f""" - Call to python script fill_jinja_template.py to create a rocoto workflow - XML file from a template file failed. Parameters passed to this script - are: - Full path to template rocoto XML file: - template_xml_fp = \"{template_xml_fp}\" - Full path to output rocoto XML file: - WFLOW_XML_FP = \"{WFLOW_XML_FP}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a rocoto workflow + XML file from a template file failed. Parameters passed to this script + are: + Full path to template rocoto XML file: + template_xml_fp = '{template_xml_fp}' + Full path to output rocoto XML file: + WFLOW_XML_FP = '{WFLOW_XML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) @@ -438,11 +438,11 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> # ----------------------------------------------------------------------- # log_info( - f''' + f""" Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): - EXPTDIR = \"{EXPTDIR}\" - WFLOW_LAUNCH_SCRIPT_FP = \"{WFLOW_LAUNCH_SCRIPT_FP}\"''', + EXPTDIR = '{EXPTDIR}' + WFLOW_LAUNCH_SCRIPT_FP = '{WFLOW_LAUNCH_SCRIPT_FP}'""", verbose=VERBOSE, ) @@ -467,21 +467,21 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> if SYMLINK_FIX_FILES: log_info( - f''' + f""" Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): - FIXgsm = \"{FIXgsm}\" - FIXam = \"{FIXam}\"''', + FIXgsm = '{FIXgsm}' + FIXam = '{FIXam}'""", verbose=VERBOSE, ) - ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') + ln_vrfy(f"""-fsn '{FIXgsm}' '{FIXam}'""") else: log_info( - f''' + f""" Copying fixed files from system directory (FIXgsm) to a subdirectory (FIXam): - FIXgsm = \"{FIXgsm}\" - FIXam = \"{FIXam}\"''', + FIXgsm = '{FIXgsm}' + FIXam = '{FIXam}'""", verbose=VERBOSE, ) @@ -502,12 +502,12 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> # if USE_MERRA_CLIMO: log_info( - f''' + f""" Copying MERRA2 aerosol climatology data files from system directory (FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: - FIXaer = \"{FIXaer}\" - FIXlut = \"{FIXlut}\" - FIXclim = \"{FIXclim}\"''', + FIXaer = '{FIXaer}' + FIXlut = '{FIXlut}' + FIXclim = '{FIXclim}'""", verbose=VERBOSE, ) @@ -585,9 +585,9 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> # ----------------------------------------------------------------------- # log_info( - f''' + f""" Setting parameters in weather model's namelist file (FV3_NML_FP): - FV3_NML_FP = \"{FV3_NML_FP}\"''' + FV3_NML_FP = '{FV3_NML_FP}'""" ) # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. @@ -818,8 +818,8 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> log_info( f""" - The variable \"settings\" specifying values of the weather model's - namelist variables has been set as follows:\n""", + The variable 'settings' specifying values of the weather model's + namelist variables has been set as follows:\n""", verbose=VERBOSE, ) log_info("\nsettings =\n\n" + settings_str, verbose=VERBOSE) @@ -855,18 +855,18 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> logging.exception( dedent( f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = \"{FV3_NML_BASE_SUITE_FP}\" - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = \"{FV3_NML_YAML_CONFIG_FP}\" - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" - Full path to output namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' + Full path to yaml configuration file for various physics suites: + FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' + Physics suite to extract from yaml configuration file: + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) @@ -919,7 +919,7 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> Experiment generation completed. The experiment directory is: - EXPTDIR=\"{EXPTDIR}\" + EXPTDIR='{EXPTDIR}' ======================================================================== ======================================================================== @@ -964,10 +964,10 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> rocotostat command. For automatic resubmission of the workflow (say every {CRON_RELAUNCH_INTVL_MNTS} minutes), the - following line can be added to the user's crontab (use \"crontab -e\" to + following line can be added to the user's crontab (use 'crontab -e' to edit the cron table): - */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" + */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron="TRUE" """ ) @@ -990,8 +990,8 @@ def get_nomads_data( NOMADS_script = os.path.join(USHdir, "NOMADS_get_extrn_mdl_files.sh") run_command( f"""{NOMADS_script} \ - {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ - {date_to_str(DATE_FIRST_CYCL,format="%H")} \ + {date_to_str(DATE_FIRST_CYCL,format='%Y%m%d')} \ + {date_to_str(DATE_FIRST_CYCL,format='%H')} \ {NOMADS_file_type} \ {FCST_LEN_HRS} \ {LBC_SPEC_INTVL_HRS}""" @@ -1029,13 +1029,13 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow") -> None: logging.exception( dedent( f""" - ********************************************************************* - FATAL ERROR: - Experiment generation failed. See the error message(s) printed below. - For more detailed information, check the log file from the workflow - generation script: {logfile} - *********************************************************************\n - """ + ********************************************************************* + FATAL ERROR: + Experiment generation failed. See the error message(s) printed below. + For more detailed information, check the log file from the workflow + generation script: {logfile} + *********************************************************************\n + """ ) ) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 98e0c369cd..a1e7863a43 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -102,9 +102,9 @@ def add_crontab_line(): time_stamp = datetime.now().strftime("%F_%T") crontab_backup_fp = os.path.join(EXPTDIR, f"crontab.bak.{time_stamp}") log_info( - f''' + f""" Copying contents of user cron table to backup file: - crontab_backup_fp = \"{crontab_backup_fp}\"''', + crontab_backup_fp = '{crontab_backup_fp}'""", verbose=VERBOSE, ) @@ -120,25 +120,25 @@ def add_crontab_line(): ) # Create backup - run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''') + run_command(f"""printf "%s" '{crontab_contents}' > '{crontab_backup_fp}'""") # Add crontab line if CRONTAB_LINE in crontab_contents: log_info( - f''' + f""" The following line already exists in the cron table and thus will not be added: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''' + CRONTAB_LINE = '{CRONTAB_LINE}'""" ) else: log_info( - f''' + f""" Adding the following line to the user's cron table in order to automatically resubmit SRW workflow: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''', + CRONTAB_LINE = '{CRONTAB_LINE}'""", verbose=VERBOSE, ) diff --git a/ush/link_fix.py b/ush/link_fix.py index 752fb7743f..5602d5170f 100755 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -230,7 +230,7 @@ def link_fix(verbose, file_group): f""" The resolution could not be extracted from the current file's name. The full path to the file (fp) is: - fp = \"{fp}\" + fp = '{fp}' This may be because fp contains the * globbing character, which would imply that no files were found that match the globbing pattern specified in fp.""" @@ -243,8 +243,8 @@ def link_fix(verbose, file_group): f""" The resolutions (as obtained from the file names) of the previous and current file (fp_prev and fp, respectively) are different: - fp_prev = \"{fp_prev}\" - fp = \"{fp}\" + fp_prev = '{fp_prev}' + fp = '{fp}' Please ensure that all files have the same resolution.""" ) diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index 1c96d0ba7c..e53b8cec1f 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -216,9 +216,9 @@ def load_ini_config(config_file, return_string=0): if not os.path.exists(config_file): raise FileNotFoundError( dedent( - f''' + f""" The specified configuration file does not exist: - "{config_file}"''' + '{config_file}'""" ) ) diff --git a/ush/python_utils/create_symlink_to_file.py b/ush/python_utils/create_symlink_to_file.py index 3a3fddb08b..363a49fa40 100644 --- a/ush/python_utils/create_symlink_to_file.py +++ b/ush/python_utils/create_symlink_to_file.py @@ -22,26 +22,26 @@ def create_symlink_to_file(target, symlink, relative=True): if target is None: print_err_msg_exit( - f''' - The argument \"target\" specifying the target of the symbolic link that + f""" + The argument 'target' specifying the target of the symbolic link that this function will create was not specified in the call to this function: - target = \"{target}\"''' + target = '{target}'""" ) if symlink is None: print_err_msg_exit( - f''' - The argument \"symlink\" specifying the target of the symbolic link that + f""" + The argument 'symlink' specifying the target of the symbolic link that this function will create was not specified in the call to this function: - symlink = \"{symlink}\"''' + symlink = '{symlink}'""" ) if not os.path.exists(target): print_err_msg_exit( - f''' + f""" Cannot create symlink to specified target file because the latter does not exist or is not a file: - target = \"{target}\"''' + target = '{target}'""" ) relative_flag = "" diff --git a/ush/python_utils/filesys_cmds_vrfy.py b/ush/python_utils/filesys_cmds_vrfy.py index 7cae27b7d5..d99b97c7f2 100644 --- a/ush/python_utils/filesys_cmds_vrfy.py +++ b/ush/python_utils/filesys_cmds_vrfy.py @@ -17,7 +17,7 @@ def cmd_vrfy(cmd, *args): cmd += " " + " ".join([str(a) for a in args]) ret = os.system(cmd) if ret != 0: - print_err_msg_exit(f'System call "{cmd}" failed.') + print_err_msg_exit(f"System call '{cmd}' failed.") return ret diff --git a/ush/python_utils/get_charvar_from_netcdf.py b/ush/python_utils/get_charvar_from_netcdf.py index c5f642b03a..8dbf798245 100644 --- a/ush/python_utils/get_charvar_from_netcdf.py +++ b/ush/python_utils/get_charvar_from_netcdf.py @@ -28,21 +28,21 @@ def get_charvar_from_netcdf(nc_file, nc_var_name): if ret != 0: print_err_msg_exit( - f''' + f""" Attempt to extract the value of the NetCDF variable spcecified by nc_var_name from the file specified by nc_file failed: - nc_file = \"{nc_file}\" - nc_var_name = \"{nc_var_name}\"''' + nc_file = '{nc_file}' + nc_var_name = '{nc_var_name}'""" ) if nc_var_value is None: print_err_msg_exit( - f''' + f""" In the specified NetCDF file (nc_file), the specified variable (nc_var_name) was not found: - nc_file = \"{nc_file}\" - nc_var_name = \"{nc_var_name}\" - nc_var_value = \"{nc_var_value}\"''' + nc_file = '{nc_file}' + nc_var_name = '{nc_var_name}' + nc_var_value = '{nc_var_value}'""" ) return nc_var_value diff --git a/ush/python_utils/print_input_args.py b/ush/python_utils/print_input_args.py index 3edcef3428..6d2a6ec2f1 100644 --- a/ush/python_utils/print_input_args.py +++ b/ush/python_utils/print_input_args.py @@ -33,23 +33,23 @@ def print_input_args(valid_args): if num_valid_args == 0: msg = dedent( - f''' + f""" No arguments have been passed to function {function} in script {filename_base} located - \"{filename}\"''' + '{filename}'""" ) else: msg = dedent( f""" The arguments to function {function} in script {filename_base} located - \"{filename}\" + '{filename}' have been set as follows:\n\n""" ) for k, v in valid_arg_names.items(): - msg = msg + f' {k}="{v}"\n' + msg = msg + f" {k}='{v}'\n" print_info_msg(msg, verbose=DEBUG) return num_valid_args diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 80baba6eb0..098aadc37f 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -101,10 +101,10 @@ def set_FV3nml_ens_stoch_seeds(cdate): print_info_msg( dedent( f""" - The variable \"settings\" specifying seeds in \"{FV3_NML_FP}\" - has been set as follows: + The variable 'settings' specifying seeds in '{FV3_NML_FP}' + has been set as follows: - settings =\n\n""" + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -118,15 +118,15 @@ def set_FV3nml_ens_stoch_seeds(cdate): print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Full path to output namelist file: - fv3_nml_ensmem_fp = \"{fv3_nml_ensmem_fp}\" - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file: + fv3_nml_ensmem_fp = '{fv3_nml_ensmem_fp}' + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n""" ) + settings_str ) @@ -181,12 +181,12 @@ def setUp(self): "-p", os.path.join( EXPTDIR, - f'{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem{i+1}', + f"{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem{i+1}", ), ) cd_vrfy( - f'{EXPTDIR}{os.sep}{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem2' + f"{EXPTDIR}{os.sep}{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem2" ) set_env_var("USHdir", USHdir) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 59c3b10401..4cad4dfff9 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -88,9 +88,9 @@ def set_FV3nml_sfc_climo_filenames(): print_info_msg( dedent( f""" - The variable \"settings\" specifying values of the namelist variables - has been set as follows:\n - settings =\n\n""" + The variable 'settings' specifying values of the namelist variables + has been set as follows:\n + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -108,15 +108,15 @@ def set_FV3nml_sfc_climo_filenames(): print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_nml_base_fp = \"{fv3_nml_base_fp}\" - Full path to output namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + fv3_nml_base_fp = '{fv3_nml_base_fp}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n""" ) + settings_str ) diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index f8536948aa..28f6e1ef68 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -97,7 +97,7 @@ def set_ozone_param( else: raise KeyError( f"Unknown or no ozone parameterization specified in the " - 'CCPP physics suite file "{ccpp_phys_suite_fp}"' + "CCPP physics suite file '{ccpp_phys_suite_fp}'" ) # # ----------------------------------------------------------------------- @@ -172,11 +172,11 @@ def set_ozone_param( else: raise Exception( - f''' + f""" Unable to set name of the ozone production/loss file in the FIXgsm directory in the array that specifies the mapping between the symlinks that need to be created in the cycle directories and the files in the FIXgsm directory: - fixgsm_ozone_fn_is_set = \"{fixgsm_ozone_fn_is_set}\"''' + fixgsm_ozone_fn_is_set = '{fixgsm_ozone_fn_is_set}'""" ) return ozone_param diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index 8cf9382947..faf8c75c58 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -43,8 +43,8 @@ def set_predef_grid_params(): except KeyError: errmsg = dedent( f""" - PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml - Check your config file settings.""" + PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml + Check your config file settings.""" ) raise Exception(errmsg) from None diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index a70bfe19f9..df69c2b232 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -107,7 +107,7 @@ def set_thompson_mp_fix_files( CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the values of these parameters are as follows: - CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' """ ) log_info( diff --git a/ush/setup.py b/ush/setup.py index 78bcafa7c0..1bdf2453a8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -68,7 +68,7 @@ def setup(): log_info( f""" ======================================================================== - Starting function setup() in \"{os.path.basename(__file__)}\"... + Starting function setup() in '{os.path.basename(__file__)}'... ========================================================================""" ) # @@ -127,7 +127,7 @@ def setup(): for val in mandatory: if val not in cfg_u: raise Exception( - f'Mandatory variable "{val}" not found in user config file {EXPT_CONFIG_FN}' + f"Mandatory variable '{val}' not found in user config file {EXPT_CONFIG_FN}" ) import_vars( @@ -155,8 +155,8 @@ def setup(): raise FileNotFoundError( dedent( f""" - The machine file {MACHINE_FILE} does not exist. - Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" + The machine file {MACHINE_FILE} does not exist. + Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" ) ) machine_cfg = load_config_file(MACHINE_FILE) @@ -255,7 +255,7 @@ def get_location(xcs, fmt): if DEBUG and not VERBOSE: log_info( """ - Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"...""" + Resetting VERBOSE to 'TRUE' because DEBUG has been set to 'TRUE'...""" ) VERBOSE = True @@ -373,7 +373,7 @@ def get_location(xcs, fmt): LSM_SPP_MAG_LIST (length {len(LSM_SPP_MAG_LIST)}) LSM_SPP_LSCALE (length {len(LSM_SPP_LSCALE)}) LSM_SPP_TSCALE (length {len(LSM_SPP_TSCALE)}) - """ + """ ) # # The current script should be located in the ush subdirectory of the @@ -414,7 +414,7 @@ def get_location(xcs, fmt): errmsg = dedent( f""" Externals configuration file {mng_extrns_cfg_fn} - does not contain "{external_name}".""" + does not contain '{external_name}'.""" ) raise Exception(errmsg) from None @@ -423,11 +423,11 @@ def get_location(xcs, fmt): raise FileNotFoundError( dedent( f""" - The base directory in which the FV3 source code should be located - (UFS_WTHR_MDL_DIR) does not exist: - UFS_WTHR_MDL_DIR = \"{UFS_WTHR_MDL_DIR}\" - Please clone the external repository containing the code in this directory, - build the executable, and then rerun the workflow.""" + The base directory in which the FV3 source code should be located + (UFS_WTHR_MDL_DIR) does not exist: + UFS_WTHR_MDL_DIR = '{UFS_WTHR_MDL_DIR}' + Please clone the external repository containing the code in this directory, + build the executable, and then rerun the workflow.""" ) ) # @@ -476,7 +476,7 @@ def get_location(xcs, fmt): raise Exception( dedent( f""" - Mandatory variable "{val}" not found in: + Mandatory variable '{val}' not found in: user config file {EXPT_CONFIG_FN} OR machine file {MACHINE_FILE} @@ -525,8 +525,8 @@ def get_location(xcs, fmt): raise Exception( dedent( f""" - ACCOUNT must be specified in config or machine file if using a workflow manager. - WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n""" + ACCOUNT must be specified in config or machine file if using a workflow manager. + WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n""" ) ) # @@ -563,9 +563,9 @@ def get_location(xcs, fmt): CPL = True else: raise Exception( - f''' + f""" The coupling flag CPL has not been specified for this value of FCST_MODEL: - FCST_MODEL = \"{FCST_MODEL}\"''' + FCST_MODEL = '{FCST_MODEL}'""" ) # Make sure RESTART_INTERVAL is set to an integer value @@ -602,9 +602,9 @@ def get_location(xcs, fmt): raise FileNotFoundError( dedent( f""" - USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file - CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} - could not be found.""" + USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file + CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} + could not be found.""" ) ) from None @@ -619,9 +619,9 @@ def get_location(xcs, fmt): raise FileNotFoundError( dedent( f""" - USE_CRTM has been set, but the external CRTM fix file directory: - CRTM_DIR = {CRTM_DIR} - could not be found.""" + USE_CRTM has been set, but the external CRTM fix file directory: + CRTM_DIR = {CRTM_DIR} + could not be found.""" ) ) from None @@ -707,12 +707,12 @@ def get_location(xcs, fmt): # if DT_SUBHOURLY_POST_MNTS < 0 or DT_SUBHOURLY_POST_MNTS > 59: raise ValueError( - f''' - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + f""" + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), DT_SUBHOURLY_POST_MNTS must be set to an integer between 0 and 59, inclusive but in this case is not: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\"''' + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}'""" ) # # Check that DT_SUBHOURLY_POST_MNTS (after converting to seconds) is @@ -722,14 +722,14 @@ def get_location(xcs, fmt): if rem != 0: raise ValueError( f""" - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), the time interval specified by DT_SUBHOURLY_POST_MNTS (after converting to seconds) must be evenly divisible by the time step DT_ATMOS used in the forecast model, i.e. the remainder (rem) must be zero. In this case, it is not: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" - DT_ATMOS = \"{DT_ATMOS}\" + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}' + DT_ATMOS = '{DT_ATMOS}' rem = (DT_SUBHOURLY_POST_MNTS*60) %% DT_ATMOS = {rem} Please reset DT_SUBHOURLY_POST_MNTS and/or DT_ATMOS so that this remainder is zero.""" @@ -743,12 +743,12 @@ def get_location(xcs, fmt): if DT_SUBHOURLY_POST_MNTS == 0: logger.warning( f""" - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), DT_SUBHOURLY_POST_MNTS must be set to a value greater than 0; otherwise, sub-hourly output is not really being performed: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" - Resetting SUB_HOURLY_POST to \"FALSE\". If you do not want this, you + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}' + Resetting SUB_HOURLY_POST to 'FALSE'. If you do not want this, you must set DT_SUBHOURLY_POST_MNTS to something other than zero.""" ) SUB_HOURLY_POST = False @@ -779,7 +779,7 @@ def get_location(xcs, fmt): pass EXPT_BASEDIR = os.path.abspath(EXPT_BASEDIR) - mkdir_vrfy(f' -p "{EXPT_BASEDIR}"') + mkdir_vrfy(f" -p '{EXPT_BASEDIR}'") # # ----------------------------------------------------------------------- @@ -935,13 +935,13 @@ def get_location(xcs, fmt): # create NCO directories if RUN_ENVIR == "nco": - mkdir_vrfy(f' -p "{OPSROOT}"') - mkdir_vrfy(f' -p "{COMROOT}"') - mkdir_vrfy(f' -p "{PACKAGEROOT}"') - mkdir_vrfy(f' -p "{DATAROOT}"') - mkdir_vrfy(f' -p "{DCOMROOT}"') + mkdir_vrfy(f" -p '{OPSROOT}'") + mkdir_vrfy(f" -p '{COMROOT}'") + mkdir_vrfy(f" -p '{PACKAGEROOT}'") + mkdir_vrfy(f" -p '{DATAROOT}'") + mkdir_vrfy(f" -p '{DCOMROOT}'") if DBNROOT is not None: - mkdir_vrfy(f' -p "{DBNROOT}"') + mkdir_vrfy(f" -p '{DBNROOT}'") # # ----------------------------------------------------------------------- @@ -965,10 +965,10 @@ def get_location(xcs, fmt): f""" The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) has not been set: - POST_OUTPUT_DOMAIN_NAME = \"{POST_OUTPUT_DOMAIN_NAME}\" + POST_OUTPUT_DOMAIN_NAME = '{POST_OUTPUT_DOMAIN_NAME}' If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME is set to a null string), POST_OUTPUT_DOMAIN_NAME - must be set in the configuration file (\"{EXPT_CONFIG_FN}\"). """ + must be set in the configuration file ('{EXPT_CONFIG_FN}'). """ ) # # ----------------------------------------------------------------------- @@ -1070,10 +1070,10 @@ def get_location(xcs, fmt): CCPP_PHYS_SUITE_FP = os.path.join(EXPTDIR, CCPP_PHYS_SUITE_FN) if not os.path.exists(CCPP_PHYS_SUITE_IN_CCPP_FP): raise FileNotFoundError( - f''' + f""" The CCPP suite definition file (CCPP_PHYS_SUITE_IN_CCPP_FP) does not exist in the local clone of the ufs-weather-model: - CCPP_PHYS_SUITE_IN_CCPP_FP = \"{CCPP_PHYS_SUITE_IN_CCPP_FP}\"''' + CCPP_PHYS_SUITE_IN_CCPP_FP = '{CCPP_PHYS_SUITE_IN_CCPP_FP}'""" ) # # ----------------------------------------------------------------------- @@ -1097,10 +1097,10 @@ def get_location(xcs, fmt): FIELD_DICT_FP = os.path.join(EXPTDIR, FIELD_DICT_FN) if not os.path.exists(FIELD_DICT_IN_UWM_FP): raise FileNotFoundError( - f''' + f""" The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist in the local clone of the ufs-weather-model: - FIELD_DICT_IN_UWM_FP = \"{FIELD_DICT_IN_UWM_FP}\"''' + FIELD_DICT_IN_UWM_FP = '{FIELD_DICT_IN_UWM_FP}'""" ) # # ----------------------------------------------------------------------- @@ -1173,10 +1173,10 @@ def get_location(xcs, fmt): if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]): raise FileNotFoundError( - f''' + f""" The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''' + EXTRN_MDL_SOURCE_BASEDIR_ICS = '{EXTRN_MDL_SOURCE_BASEDIR_ICS}'""" ) idx = EXTRN_MDL_SOURCE_BASEDIR_LBCS.find("$") @@ -1185,10 +1185,10 @@ def get_location(xcs, fmt): if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): raise FileNotFoundError( - f''' + f""" The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_LBCS = \"{EXTRN_MDL_SOURCE_BASEDIR_LBCS}\"''' + EXTRN_MDL_SOURCE_BASEDIR_LBCS = '{EXTRN_MDL_SOURCE_BASEDIR_LBCS}'""" ) # # ----------------------------------------------------------------------- @@ -1263,11 +1263,11 @@ def get_location(xcs, fmt): # Ensemble verification can only be run in ensemble mode if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): raise Exception( - f''' + f""" Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{DO_ENSEMBLE}\" - RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" - RUN_TASK_VX_ENSPOINT = \"{RUN_TASK_VX_ENSPOINT}\"''' + DO_ENSEMBLE = '{DO_ENSEMBLE}' + RUN_TASK_VX_ENSGRID = '{RUN_TASK_VX_ENSGRID}' + RUN_TASK_VX_ENSPOINT = '{RUN_TASK_VX_ENSPOINT}'""" ) # @@ -1313,18 +1313,18 @@ def get_location(xcs, fmt): msg = dedent( f""" - GRID_DIR not specified! - Setting GRID_DIR = {GRID_DIR} - """ + GRID_DIR not specified! + Setting GRID_DIR = {GRID_DIR} + """ ) logger.warning(msg) if not os.path.exists(GRID_DIR): raise FileNotFoundError( - f''' + f""" The directory (GRID_DIR) that should contain the pregenerated grid files does not exist: - GRID_DIR = \"{GRID_DIR}\"''' + GRID_DIR = '{GRID_DIR}'""" ) else: GRID_DIR = os.path.join(EXPTDIR, "grid") @@ -1340,18 +1340,18 @@ def get_location(xcs, fmt): msg = dedent( f""" - OROG_DIR not specified! - Setting OROG_DIR = {OROG_DIR} - """ + OROG_DIR not specified! + Setting OROG_DIR = {OROG_DIR} + """ ) logger.warning(msg) if not os.path.exists(OROG_DIR): raise FileNotFoundError( - f''' + f""" The directory (OROG_DIR) that should contain the pregenerated orography files does not exist: - OROG_DIR = \"{OROG_DIR}\"''' + OROG_DIR = '{OROG_DIR}'""" ) else: OROG_DIR = os.path.join(EXPTDIR, "orog") @@ -1367,18 +1367,18 @@ def get_location(xcs, fmt): msg = dedent( f""" - SFC_CLIMO_DIR not specified! - Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} - """ + SFC_CLIMO_DIR not specified! + Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} + """ ) logger.warning(msg) if not os.path.exists(SFC_CLIMO_DIR): raise FileNotFoundError( - f''' + f""" The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface climatology files does not exist: - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''' + SFC_CLIMO_DIR = '{SFC_CLIMO_DIR}'""" ) else: SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") @@ -1470,8 +1470,8 @@ def get_location(xcs, fmt): # # ----------------------------------------------------------------------- # - mkdir_vrfy(f' -p "{EXPTDIR}"') - mkdir_vrfy(f' -p "{LOGDIR}"') + mkdir_vrfy(f" -p '{EXPTDIR}'") + mkdir_vrfy(f" -p '{LOGDIR}'") # # ----------------------------------------------------------------------- # NOTE: currently this is executed no matter what, should it be dependent on the logic described below?? @@ -1484,7 +1484,7 @@ def get_location(xcs, fmt): # # ----------------------------------------------------------------------- # - mkdir_vrfy(f' -p "{FIXlam}"') + mkdir_vrfy(f" -p '{FIXlam}'") RES_IN_FIXLAM_FILENAMES = "" # # ----------------------------------------------------------------------- @@ -1608,9 +1608,9 @@ def get_location(xcs, fmt): if VERBOSE: log_info( f""" - The number of MPI tasks for the forecast (including those for the write - component if it is being used) are: - PE_MEMBER01 = {PE_MEMBER01}""", + The number of MPI tasks for the forecast (including those for the write + component if it is being used) are: + PE_MEMBER01 = {PE_MEMBER01}""", verbose=VERBOSE, ) # @@ -1715,7 +1715,7 @@ def get_location(xcs, fmt): # # Full path to workflow (re)launch script, its log file, and the line # that gets added to the cron table to launch this script if the flag - # USE_CRON_TO_RELAUNCH is set to \"TRUE\". + # USE_CRON_TO_RELAUNCH is set to 'TRUE'. # # ----------------------------------------------------------------------- # @@ -1836,7 +1836,7 @@ def get_location(xcs, fmt): # # ----------------------------------------------------------------------- # - # Flag in the \"{MODEL_CONFIG_FN}\" file for coupling the ocean model to + # Flag in the '{MODEL_CONFIG_FN}' file for coupling the ocean model to # the weather model. # # ----------------------------------------------------------------------- @@ -1925,11 +1925,11 @@ def get_location(xcs, fmt): f""" Generating the global experiment variable definitions file specified by GLOBAL_VAR_DEFNS_FN: - GLOBAL_VAR_DEFNS_FN = \"{GLOBAL_VAR_DEFNS_FN}\" + GLOBAL_VAR_DEFNS_FN = '{GLOBAL_VAR_DEFNS_FN}' Full path to this file is: - GLOBAL_VAR_DEFNS_FP = \"{GLOBAL_VAR_DEFNS_FP}\" - For more detailed information, set DEBUG to \"TRUE\" in the experiment - configuration file (\"{EXPT_CONFIG_FN}\").""" + GLOBAL_VAR_DEFNS_FP = '{GLOBAL_VAR_DEFNS_FP}' + For more detailed information, set DEBUG to 'TRUE' in the experiment + configuration file ('{EXPT_CONFIG_FN}').""" ) with open(GLOBAL_VAR_DEFNS_FP, "a") as f: From 12c41c61d10df042edae5988f464f02a0147c16a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 25 Oct 2022 20:12:28 +0000 Subject: [PATCH 9/9] Removed unused python utilities. --- ush/python_utils/__init__.py | 4 -- ush/python_utils/count_files.py | 16 ------- ush/python_utils/get_charvar_from_netcdf.py | 48 --------------------- ush/python_utils/get_elem_inds.py | 28 ------------ ush/python_utils/interpol_to_arbit_CRES.py | 43 ------------------ ush/python_utils/test_python_utils.py | 23 ---------- 6 files changed, 162 deletions(-) delete mode 100644 ush/python_utils/count_files.py delete mode 100644 ush/python_utils/get_charvar_from_netcdf.py delete mode 100644 ush/python_utils/get_elem_inds.py delete mode 100644 ush/python_utils/interpol_to_arbit_CRES.py diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index add96ee981..02e2b83b77 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -1,7 +1,6 @@ from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file from .check_for_preexist_dir_file import check_for_preexist_dir_file from .check_var_valid_value import check_var_valid_value -from .count_files import count_files from .create_symlink_to_file import create_symlink_to_file from .define_macos_utilities import define_macos_utilities from .environment import ( @@ -25,12 +24,9 @@ mkdir_vrfy, cd_vrfy, ) -from .get_elem_inds import get_elem_inds -from .interpol_to_arbit_CRES import interpol_to_arbit_CRES from .print_input_args import print_input_args from .print_msg import print_info_msg, print_err_msg_exit, log_info from .run_command import run_command -from .get_charvar_from_netcdf import get_charvar_from_netcdf from .xml_parser import load_xml_file, has_tag_with_value from .config_parser import ( load_json_config, diff --git a/ush/python_utils/count_files.py b/ush/python_utils/count_files.py deleted file mode 100644 index 36862895b7..0000000000 --- a/ush/python_utils/count_files.py +++ /dev/null @@ -1,16 +0,0 @@ -import glob - - -def count_files(ext, dirct="."): - """Function that returns the number of files in the specified directory - ending with the specified file extension - - Args: - ext: File extension string - dir: Directory to parse (default is current directory) - Returns: - int: Number of files - """ - - files = glob.glob(dirct + "/*." + ext) - return len(files) diff --git a/ush/python_utils/get_charvar_from_netcdf.py b/ush/python_utils/get_charvar_from_netcdf.py deleted file mode 100644 index 8dbf798245..0000000000 --- a/ush/python_utils/get_charvar_from_netcdf.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import os -from .print_msg import print_err_msg_exit -from .run_command import run_command - - -def get_charvar_from_netcdf(nc_file, nc_var_name): - """Searches NetCDF file and extract a scalar variable - - Args: - nc_file: Path to netCDF file - nc_var_name: name of the scalar variable - Returns: - value of the variable - """ - - SED = os.getenv("SED") - - cmd = f"ncdump -v {nc_var_name} {nc_file} | \ - {SED} -r -e '1,/data:/d' \ - -e '/^[ ]*'{nc_var_name}'/d' \ - -e '/^}}$/d' \ - -e 's/.*\"(.*)\".*/\\1/' \ - -e '/^$/d' \ - " - (ret, nc_var_value, _) = run_command(cmd) - - if ret != 0: - print_err_msg_exit( - f""" - Attempt to extract the value of the NetCDF variable spcecified by nc_var_name - from the file specified by nc_file failed: - nc_file = '{nc_file}' - nc_var_name = '{nc_var_name}'""" - ) - - if nc_var_value is None: - print_err_msg_exit( - f""" - In the specified NetCDF file (nc_file), the specified variable (nc_var_name) - was not found: - nc_file = '{nc_file}' - nc_var_name = '{nc_var_name}' - nc_var_value = '{nc_var_value}'""" - ) - - return nc_var_value diff --git a/ush/python_utils/get_elem_inds.py b/ush/python_utils/get_elem_inds.py deleted file mode 100644 index 20ac2e8967..0000000000 --- a/ush/python_utils/get_elem_inds.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 - -from .misc import lowercase -from .check_var_valid_value import check_var_valid_value - - -def get_elem_inds(arr, match, ret_type): - """Function that returns indices of elements of array - that match a given string - - Args: - arr: the list - match: element to match (case insenensitive) - ret_type: the return type can be any of [ 'first', 'last', 'all' ] - Returns: - A list of indices - """ - ret_type = lowercase(ret_type) - check_var_valid_value(ret_type, ["first", "last", "all"]) - - if ret_type == "first": - return arr.index(match) - if ret_type == "last": - for i in range(len(arr) - 1, -1, -1): - if arr[i] == match: - return i - return None - return [i for i, e in enumerate(arr) if e == match] diff --git a/ush/python_utils/interpol_to_arbit_CRES.py b/ush/python_utils/interpol_to_arbit_CRES.py deleted file mode 100644 index d87e258000..0000000000 --- a/ush/python_utils/interpol_to_arbit_CRES.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - - -def interpol_to_arbit_CRES(RES, RES_array, prop_array): - """Function to interpolate (or extrapolate) a grid cell size-dependent property - to an arbitrary cubed-sphere resolution using arrays that specify a set of property - values for a corresponding set of resolutions - - Args: - RES: The cubed-sphere resolution at which to find the value of a property. - This is in units of number of cells (in either of the two horizontal - directions) on any one of the tiles of a cubed-sphere grid. - - RES_array: The name of the array containing the cubed-sphere resolutions for - which corresponding property values are given (in prop_array). These - are assumed to be given from smallest to largest. - - prop_array: The name of the array containing the values of the property corres- - ponding to the cubed-sphere resolutions in RES_array. - Returns: - Interpolated (extrapolated) property value - """ - - num_valid_RESes = len(RES_array) - i_min = 0 - i_max = num_valid_RESes - 1 - - if RES <= RES_array[i_min]: - prop = prop_array[i_min] - elif RES > RES_array[i_max]: - prop = prop_array[i_max] - else: - for i in range(0, num_valid_RESes - 1): - if RES_array[i] < RES <= RES_array[i + 1]: - RES1 = RES_array[i] - RES2 = RES_array[i + 1] - prop1 = prop_array[i] - prop2 = prop_array[i + 1] - m_slope = (prop2 - prop1) / (RES2 - RES1) - y_intcpt = (RES2 * prop1 - RES1 * prop2) / (RES2 - RES1) - prop = m_slope * RES + y_intcpt - - return prop diff --git a/ush/python_utils/test_python_utils.py b/ush/python_utils/test_python_utils.py index 7e25fdaf5b..69b6daf1a2 100644 --- a/ush/python_utils/test_python_utils.py +++ b/ush/python_utils/test_python_utils.py @@ -49,11 +49,6 @@ def test_check_for_preexist_dir_file(self): def test_check_var_valid_value(self): self.assertTrue(check_var_valid_value("rice", ["egg", "spam", "rice"])) - def test_count_files(self): - (_, target_cnt, _) = run_command("ls -l *.py | wc -l") - cnt = count_files("py") - self.assertEqual(cnt, int(target_cnt)) - def test_filesys_cmds(self): dPATH = f"{self.PATH}/test_data/dir" mkdir_vrfy(dPATH) @@ -63,27 +58,9 @@ def test_filesys_cmds(self): cmd_vrfy(f"rm -rf {dPATH}") self.assertFalse(os.path.exists("tt.py")) - def test_get_charvar_from_netcdf(self): - FILE = f"{self.PATH}/test_data/sample.nc" - val = get_charvar_from_netcdf(FILE, "pressure") - self.assertTrue(val and (val.split()[0], "955.5,")) - def test_run_command(self): self.assertEqual(run_command("echo hello"), (0, "hello", "")) - def test_get_elem_inds(self): - arr = ["egg", "spam", "egg", "rice", "egg"] - self.assertEqual(get_elem_inds(arr, "egg", "first"), 0) - self.assertEqual(get_elem_inds(arr, "egg", "last"), 4) - self.assertEqual(get_elem_inds(arr, "egg", "all"), [0, 2, 4]) - - def test_interpol_to_arbit_CRES(self): - RES = 800 - RES_array = [5, 25, 40, 60, 80, 100, 400, 700, 1000, 1500, 2800, 3000] - prop_array = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.65, 0.7, 1.0, 1.1, 1.2, 1.3] - prop = interpol_to_arbit_CRES(RES, RES_array, prop_array) - self.assertAlmostEqual(prop, 0.8) - def test_create_symlink_to_file(self): TARGET = f"{self.PATH}/test_python_utils.py" SYMLINK = f"{self.PATH}/test_data/test_python_utils.py"