rrfs_ci: Merge in authoritative develop

.github/PULL_REQUEST_TEMPLATE

@@ -19,7 +19,7 @@ Explicitly state what tests were run on these changes, or if any are still pendi
 ## DEPENDENCIES:
 Add any links to external PRs (e.g. regional_workflow and/or UFS PRs). For example:
 - NOAA-EMC/regional_workflow/pull/<pr_number>
-- NOAA-EMC/UFS_UTILS/pull/<pr_number>
+- ufs-community/UFS_UTILS/pull/<pr_number>
 - ufs-community/ufs-weather-model/pull/<pr_number>
 
 ## DOCUMENTATION:

Externals.cfg

@@ -8,7 +8,7 @@ required = True
 
 [ufs_utils]
 protocol = git
-repo_url = https://github.com/NOAA-EMC/UFS_UTILS
+repo_url = https://github.com/ufs-community/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = ea821358

docs/UsersGuide/source/CodeReposAndDirs.rst

@@ -33,7 +33,7 @@ repositories associated with this umbrella repo (see :numref:`Table %s <top_leve
    | Repository for the regional     | https://github.com/NOAA-EMC/regional_workflow           |
    | workflow                        |                                                         |
    +---------------------------------+---------------------------------------------------------+
-   | Repository for UFS utilities,   | https://github.com/NOAA-EMC/UFS_UTILS                   |
+   | Repository for UFS utilities,   | https://github.com/ufs-community/UFS_UTILS              |
    | including pre-processing,       |                                                         |
    | chgres_cube, and more           |                                                         |
    +---------------------------------+---------------------------------------------------------+

env/build_hera_intel.env

@@ -26,7 +26,8 @@ module load g2tmpl/1.10.0
 module load ip/3.3.3
 module load nemsio/2.5.2
 module load sp/2.3.3
-module load w3emc/2.7.3
+module load w3emc/2.9.1
+module load sigio/2.3.2
 module load w3nco/2.4.1
 module load upp/10.0.6
 

env/build_orion_intel.env

@@ -3,35 +3,36 @@
 module load contrib noaatools
 
 module load cmake/3.18.1
-module load python/3.7.5
+module load python/3.9.2
 
 module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
 
 module load hpc/1.1.0
 module load hpc-intel/2018.4
 module load hpc-impi/2018.4
 
-module load jasper/2.0.22
+module load bacio/2.4.1
+module load g2/3.4.3
+module load g2tmpl/1.10.0
+module load ip/3.3.3
+module load sp/2.3.3
+module load w3nco/2.4.1
+module load w3emc/2.9.1
+module load crtm/2.3.0
+module load upp/10.0.10
+
+module load jasper/2.0.25
 module load zlib/1.2.11
 module load png/1.6.35
 module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
 module load esmf/8_1_1
 module load fms/2020.04.03
-module load bacio/2.4.1
-module load crtm/2.3.0
-module load g2/3.4.1
-module load g2tmpl/1.9.1
-module load ip/3.3.3
 module load nemsio/2.5.2
-module load sp/2.3.3
-module load w3emc/2.7.3
-module load w3nco/2.4.1
-module load upp/10.0.6
-
 module load gfsio/1.4.1
 module load sfcio/1.4.1
+module load sigio/2.3.2
 module load landsfcutil/2.4.1
 module load nemsiogfs/2.5.3
 module load wgrib2/2.0.8

env/build_wcoss_cray_intel.env

@@ -15,7 +15,6 @@ module use /usrx/local/dev/modulefiles
 module load NetCDF-intel-sandybridge/4.7.4
 module load HDF5-parallel-intel-sandybridge/1.10.6
 
-## WCOSS cray for WW3
 module load jasper-gnu-sandybridge/1.900.1
 module load zlib-intel-sandybridge/1.2.7
 module load png-intel-sandybridge/1.2.49
@@ -40,7 +39,9 @@ module load sp/2.3.3
 module load upp/10.0.6
 module load w3emc/2.7.3
 module load w3nco/2.4.1
-module load wgrib2/2.0.8
+
+module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles
+module load wgrib2-intel/2.0.8
 
 module swap pmi pmi/5.0.11
 

env/wflow_jet.env

@@ -1,5 +1,7 @@
 # Python environment for workflow on Jet
 
+module load rocoto
+
 module use -a /contrib/miniconda3/modulefiles
 module load miniconda3
 conda activate regional_workflow

env/wflow_orion.env

@@ -1,7 +1,7 @@
 # Python environment for workflow on Orion
 
-module load rocoto
+module load contrib rocoto
 
 module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles
-module load miniconda3
+module load miniconda3/3.8
 conda activate regional_workflow

env/wflow_wcoss_cray.env

@@ -1,5 +1,9 @@
 # Python environment for workflow on WCOSS_cray
 
+module load xt-lsfhpc/9.1.3
+module use -a /usrx/local/emc_rocoto/modulefiles
+module load rocoto/1.3.0rc2
+
 module unload python/2.7.14
 module load python/3.6.3
 module use /usrx/local/nceplibs/modulefiles 

env/wflow_wcoss_dell_p3.env

@@ -1,5 +1,9 @@
 # Python environment for workflow on WCOSS_dell_p3
 
+module load lsf/10.1
+module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/
+module load ruby/2.5.1 rocoto/1.3.0rc2
+
 module load python/3.6.3
 module use /usrx/local/nceplibs/dev/modulefiles 
 module load srw-app-python/1.0.0

manage_externals/.travis.yml

@@ -1,7 +1,6 @@
 language: python
 os: linux
-python: 
-  - "2.7"
+python:
   - "3.4"
   - "3.5"
   - "3.6"

manage_externals/manic/checkout.py

@@ -380,19 +380,19 @@ def main(args):
 
     if args.status:
         # user requested status-only
-        for comp in sorted(tree_status.keys()):
+        for comp in sorted(tree_status):
             tree_status[comp].log_status_message(args.verbose)
     else:
         # checkout / update the external repositories.
         safe_to_update = check_safe_to_update_repos(tree_status)
         if not safe_to_update:
             # print status
-            for comp in sorted(tree_status.keys()):
+            for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
             msg = """The external repositories labeled with 'M' above are not in a clean state.
 
-The following are three options for how to proceed:
+The following are four options for how to proceed:
 
 (1) Go into each external that is not in a clean state and issue either a 'git status' or
     an 'svn status' command (depending on whether the external is managed by git or
@@ -412,12 +412,17 @@ def main(args):
     {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
     update the other, non-excluded externals.
 
+(4) As a last resort, if you are confident that there is no work that needs to be saved
+    from a given external, you can remove that external (via "rm -rf [directory]") and
+    then rerun the {program_name} tool. This option is mainly useful as a workaround for
+    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
+
 
 The external repositories labeled with '?' above are not under version
 control using the expected protocol. If you are sure you want to switch
 protocols, and you don't have any work you need to save from this
-directory, then run "rm -rf [directory]" before re-running the
-checkout_externals tool.
+directory, then run "rm -rf [directory]" before rerunning the
+{program_name} tool.
 """.format(program_name=program_name, config_file=args.externals)
 
             printlog('-' * 70)

manage_externals/manic/externals_description.py

@@ -193,6 +193,9 @@ def parse_submodules_desc_section(section_items, file_path):
 def read_gitmodules_file(root_dir, file_name):
     # pylint: disable=deprecated-method
     # Disabling this check because the method is only used for python2
+    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-branches
+    # pylint: disable=too-many-statements
     """Read a .gitmodules file and convert it to be compatible with an
     externals description.
     """
@@ -253,9 +256,21 @@ def read_gitmodules_file(root_dir, file_name):
                                           ExternalsDescription.REPO_URL, url)
                 externals_description.set(sec_name,
                                           ExternalsDescription.REQUIRED, 'True')
-                git_hash = submods[sec_name]['hash']
-                externals_description.set(sec_name,
-                                          ExternalsDescription.HASH, git_hash)
+                if sec_name in submods:
+                    submod_name = sec_name
+                else:
+                    # The section name does not have to match the path
+                    submod_name = path
+
+                if submod_name in submods:
+                    git_hash = submods[submod_name]['hash']
+                    externals_description.set(sec_name,
+                                              ExternalsDescription.HASH,
+                                              git_hash)
+                else:
+                    emsg = "submodule status has no section, '{}'"
+                    emsg += "\nCheck section names in externals config file"
+                    fatal_error(emsg.format(submod_name))
 
         # Required items
         externals_description.add_section(DESCRIPTION_SECTION)

manage_externals/manic/repository_git.py

@@ -109,35 +109,34 @@ def _clone_repo(self, base_dir_path, repo_dir_name, verbosity):
     def _current_ref(self):
         """Determine the *name* associated with HEAD.
 
-        If we're on a branch, then returns the branch name; otherwise,
-        if we're on a tag, then returns the tag name; otherwise, returns
+        If we're on a tag, then returns the tag name; otherwise, returns
         the current hash. Returns an empty string if no reference can be
         determined (e.g., if we're not actually in a git repository).
+
+        If we're on a branch, then the branch name is also included in
+        the returned string (in addition to the tag / hash).
         """
         ref_found = False
 
-        # If we're on a branch, then use that as the current ref
-        branch_found, branch_name = self._git_current_branch()
-        if branch_found:
-            current_ref = branch_name
+        # If we're exactly at a tag, use that as the current ref
+        tag_found, tag_name = self._git_current_tag()
+        if tag_found:
+            current_ref = tag_name
             ref_found = True
 
-        if not ref_found:
-            # Otherwise, if we're exactly at a tag, use that as the
-            # current ref
-            tag_found, tag_name = self._git_current_tag()
-            if tag_found:
-                current_ref = tag_name
-                ref_found = True
-
         if not ref_found:
             # Otherwise, use current hash as the current ref
             hash_found, hash_name = self._git_current_hash()
             if hash_found:
                 current_ref = hash_name
                 ref_found = True
 
-        if not ref_found:
+        if ref_found:
+            # If we're on a branch, include branch name in current ref
+            branch_found, branch_name = self._git_current_branch()
+            if branch_found:
+                current_ref = "{} (branch {})".format(current_ref, branch_name)
+        else:
             # If we still can't find a ref, return empty string. This
             # can happen if we're not actually in a git repo
             current_ref = ''

manage_externals/manic/sourcetree.py

@@ -331,12 +331,14 @@ def checkout(self, verbosity, load_all, load_comp=None):
             printlog('Checking out externals: ', end='')
 
         if load_all:
-            load_comps = self._all_components.keys()
+            tmp_comps = self._all_components.keys()
         elif load_comp is not None:
-            load_comps = [load_comp]
+            tmp_comps = [load_comp]
         else:
-            load_comps = self._required_compnames
-
+            tmp_comps = self._required_compnames
+        # Sort by path so that if paths are nested the
+        # parent repo is checked out first.
+        load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path())
         # checkout the primary externals
         for comp in load_comps:
             if verbosity < VERBOSITY_VERBOSE:
@@ -346,8 +348,6 @@ def checkout(self, verbosity, load_all, load_comp=None):
                 # output a newline
                 printlog(EMPTY_STR)
             self._all_components[comp].checkout(verbosity, load_all)
-        printlog('')
-
-        # now give each external an opportunitity to checkout it's externals.
-        for comp in load_comps:
+            # now give each external an opportunitity to checkout it's externals.
             self._all_components[comp].checkout_externals(verbosity, load_all)
+        printlog('')