Porting UPP for gfs v16 on wcoss2

jobs/JGLOBAL_ATMOS_NCEPPOST

@@ -128,7 +128,7 @@ export SLEEP_INT=5
 # Run relevant exglobal script
 env
 msg="HAS BEGUN on `hostname`"
-postmsg "$jlogfile" "$msg"
+postmsg "$msg"
 $LOGSCRIPT
 
 
@@ -148,7 +148,7 @@ if [ -e "$pgmout" ]; then
 fi
 
 msg="ENDED NORMALLY."
-postmsg "$jlogfile" "$msg"
+postmsg "$msg"
 
 
 ##########################################

jobs/J_NCEPPOST

@@ -27,7 +27,7 @@ export KEEPDATA=${KEEPDATA:-NO}
 ####################################
 # File To Log Msgs
 ####################################
-export jlogfile=${jlogfile:-${COMROOT}/logs/jlogfiles/jlogfile.${job}.${pid}}
+#export jlogfile=${jlogfile:-${COMROOT}/logs/jlogfiles/jlogfile.${job}.${pid}}
 
 ####################################
 # Determine Job Output Name on System

manage_externals/.travis.yml

@@ -1,25 +1,12 @@
-# NOTE(bja, 2017-11) travis-ci dosen't support python language builds
-#  on mac os. As a work around, we use built-in python on linux, and
-#  declare osx a 'generic' language, and create our own python env.
-
 language: python
 os: linux
 python: 
   - "2.7"
   - "3.4"
   - "3.5"
   - "3.6"
-matrix:
-  include:
-  - os: osx
-    language: generic
-    before_install:
-      # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough!
-      # - brew update
-      # - brew outdated python2 || brew upgrade python2
-      - pip install virtualenv
-      - virtualenv env -p python2
-      - source env/bin/activate
+  - "3.7"
+  - "3.8"
 install:
   - pip install -r test/requirements.txt
 before_script:

manage_externals/README.md

@@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     description file:
 
         $ cd ${SRC_ROOT}
-        $ ./manage_externals/checkout_externals --excernals my-externals.cfg
+        $ ./manage_externals/checkout_externals --externals my-externals.cfg
 
   * Status summary of the repositories managed by checkout_externals:
 
@@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines begining with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manage_externals/manic/checkout.py

@@ -227,6 +227,21 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines beginning with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manage_externals/manic/externals_description.py

@@ -351,6 +351,7 @@ class ExternalsDescription(dict):
     REPO_URL = 'repo_url'
     REQUIRED = 'required'
     TAG = 'tag'
+    SPARSE = 'sparse'
 
     PROTOCOL_EXTERNALS_ONLY = 'externals_only'
     PROTOCOL_GIT = 'git'
@@ -374,6 +375,7 @@ class ExternalsDescription(dict):
                              TAG: 'string',
                              BRANCH: 'string',
                              HASH: 'string',
+                             SPARSE: 'string',
                             }
                      }
 
@@ -562,6 +564,8 @@ def _check_optional(self):
                 self[field][self.REPO][self.HASH] = EMPTY_STR
             if self.REPO_URL not in self[field][self.REPO]:
                 self[field][self.REPO][self.REPO_URL] = EMPTY_STR
+            if self.SPARSE not in self[field][self.REPO]:
+                self[field][self.REPO][self.SPARSE] = EMPTY_STR
 
             # from_submodule has a complex relationship with other fields
             if self.SUBMODULE in self[field]:

manage_externals/manic/repository.py

@@ -21,6 +21,7 @@ def __init__(self, component_name, repo):
         self._branch = repo[ExternalsDescription.BRANCH]
         self._hash = repo[ExternalsDescription.HASH]
         self._url = repo[ExternalsDescription.REPO_URL]
+        self._sparse = repo[ExternalsDescription.SPARSE]
 
         if self._url is EMPTY_STR:
             fatal_error('repo must have a URL')

manage_externals/manic/repository_git.py

@@ -316,8 +316,11 @@ def _checkout_ref(self, repo_dir, verbosity, submodules):
         else:
             self._checkout_external_ref(verbosity, submodules)
 
+        if self._sparse:
+            self._sparse_checkout(repo_dir, verbosity)
         os.chdir(cwd)
 
+
     def _checkout_local_ref(self, verbosity, submodules):
         """Checkout the reference considering the local repo only. Do not
         fetch any additional remotes or specify the remote when
@@ -362,6 +365,20 @@ def _checkout_external_ref(self, verbosity, submodules):
             ref = '{0}/{1}'.format(remote_name, ref)
         self._git_checkout_ref(ref, verbosity, submodules)
 
+    def _sparse_checkout(self, repo_dir, verbosity):
+        """Use git read-tree to thin the working tree."""
+        cwd = os.getcwd()
+
+        cmd = ['cp', self._sparse, os.path.join(repo_dir,
+                                                '.git/info/sparse-checkout')]
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        os.chdir(repo_dir)
+        self._git_sparse_checkout(verbosity)
+
+        os.chdir(cwd)
+
     def _check_for_valid_ref(self, ref, remote_name=None):
         """Try some basic sanity checks on the user supplied reference so we
         can provide a more useful error message than calledprocess
@@ -776,6 +793,18 @@ def _git_checkout_ref(ref, verbosity, submodules):
         if submodules:
             GitRepository._git_update_submodules(verbosity)
 
+    @staticmethod
+    def _git_sparse_checkout(verbosity):
+        """Configure repo via read-tree."""
+        cmd = ['git', 'config', 'core.sparsecheckout', 'true']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        cmd = ['git', 'read-tree', '-mu', 'HEAD']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+
     @staticmethod
     def _git_update_submodules(verbosity):
         """Run git submodule update for the side effect of updating this

manage_externals/manic/repository_svn.py

@@ -220,9 +220,8 @@ def xml_status_is_dirty(svn_output):
                 continue
             if item == SVN_UNVERSIONED:
                 continue
-            else:
-                is_dirty = True
-                break
+            is_dirty = True
+            break
         return is_dirty
 
     # ----------------------------------------------------------------

manage_externals/manic/sourcetree.py

@@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry):
         self._externals = EMPTY_STR
         self._externals_sourcetree = None
         self._stat = ExternalStatus()
+        self._sparse = None
         # Parse the sub-elements
 
         # _path : local path relative to the containing source tree
@@ -298,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR):
         for comp in load_comps:
             printlog('{0}, '.format(comp), end='')
             stat = self._all_components[comp].status()
+            stat_final = {}
             for name in stat.keys():
                 # check if we need to append the relative_path_base to
                 # the path so it will be sorted in the correct order.
-                if not stat[name].path.startswith(relative_path_base):
-                    stat[name].path = os.path.join(relative_path_base,
-                                                   stat[name].path)
-                    # store under key = updated path, and delete the
-                    # old key.
-                    comp_stat = stat[name]
-                    del stat[name]
-                    stat[comp_stat.path] = comp_stat
-            summary.update(stat)
+                if stat[name].path.startswith(relative_path_base):
+                    # use as is, without any changes to path
+                    stat_final[name] = stat[name]
+                else:
+                    # append relative_path_base to path and store under key = updated path
+                    modified_path = os.path.join(relative_path_base,
+                                                 stat[name].path)
+                    stat_final[modified_path] = stat[name]
+                    stat_final[modified_path].path = modified_path
+            summary.update(stat_final)
 
         return summary
 

manage_externals/test/repos/simple-ext.git/refs/heads/master

@@ -1 +1 @@
-9b75494003deca69527bb64bcaa352e801611dd2
+607ec299c17dd285c029edc41a0109e49d441380

manage_externals/test/test_sys_checkout.py

@@ -88,6 +88,8 @@
 
 SVN_TEST_REPO = 'https://github.com/escomp/cesm'
 
+# Disable too-many-public-methods error
+# pylint: disable=R0904
 
 def setUpModule():  # pylint: disable=C0103
     """Setup for all tests in this module. It is called once per module!
@@ -183,6 +185,25 @@ def container_simple_svn(self, dest_dir):
 
         self.write_config(dest_dir)
 
+    def container_sparse(self, dest_dir):
+        """Create a container with a full external and a sparse external
+
+        """
+        # Create a file for a sparse pattern match
+        sparse_filename = 'sparse_checkout'
+        with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile:
+            sfile.write('readme.txt')
+
+        self.create_config()
+        self.create_section(SIMPLE_REPO_NAME, 'simp_tag',
+                            tag='tag2')
+
+        sparse_relpath = '../../{}'.format(sparse_filename)
+        self.create_section(SIMPLE_REPO_NAME, 'simp_sparse',
+                            tag='tag2', sparse=sparse_relpath)
+
+        self.write_config(dest_dir)
+
     def mixed_simple_base(self, dest_dir):
         """Create a mixed-use base externals file with only simple externals.
 
@@ -239,7 +260,8 @@ def create_metadata(self):
 
     def create_section(self, repo_type, name, tag='', branch='',
                        ref_hash='', required=True, path=EXTERNALS_NAME,
-                       externals='', repo_path=None, from_submodule=False):
+                       externals='', repo_path=None, from_submodule=False,
+                       sparse=''):
         # pylint: disable=too-many-branches
         """Create a config section with autofilling some items and handling
         optional items.
@@ -287,6 +309,9 @@ def create_section(self, repo_type, name, tag='', branch='',
         if externals:
             self._config.set(name, ExternalsDescription.EXTERNALS, externals)
 
+        if sparse:
+            self._config.set(name, ExternalsDescription.SPARSE, sparse)
+
         if from_submodule:
             self._config.set(name, ExternalsDescription.SUBMODULE, "True")
 
@@ -710,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/mixed_req'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
 
+    def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_empty_default_required(tree, name)
+
+    def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_ok_clean_required(tree, name)
+
     # ----------------------------------------------------------------
     #
     # Check results for groups of externals under specific conditions
@@ -870,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree):
         self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME)
         self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH)
 
+    def _check_container_sparse_pre_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_empty(tree)
+        self._check_simple_sparse_empty(tree)
+
+    def _check_container_sparse_post_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_ok(tree)
+        self._check_simple_sparse_ok(tree)
+
+    def _check_file_exists(self, repo_dir, pathname):
+        "Check that <pathname> exists in <repo_dir>"
+        self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname)))
+
+    def _check_file_absent(self, repo_dir, pathname):
+        "Check that <pathname> does not exist in <repo_dir>"
+        self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname)))
 
 class TestSysCheckout(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals
@@ -1234,6 +1284,14 @@ def test_container_full(self):
                                                 self.status_args)
         self._check_container_full_post_checkout(overall, tree)
 
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
         # update the mixed-use repo to point to different branch
         self._generator.update_branch(under_test_dir, 'mixed_req',
                                       'new-feature', MIXED_REPO_NAME)
@@ -1314,6 +1372,40 @@ def test_mixed_simple(self):
                                                 self.status_args)
         self._check_mixed_cont_simple_required_post_checkout(overall, tree)
 
+    def test_container_sparse(self):
+        """Verify that 'full' container with simple subrepo
+        can run a sparse checkout and generate the correct initial status.
+
+        """
+        # create the test repository
+        under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME)
+
+        # create the top level externals file
+        self._generator.container_sparse(under_test_dir)
+
+        # inital checkout
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.checkout_args)
+        self._check_container_sparse_pre_checkout(overall, tree)
+
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.status_args)
+        self._check_container_sparse_post_checkout(overall, tree)
+
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_exists(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+        subrepo_path = os.path.join('externals', 'simp_sparse')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
 
 class TestSysCheckoutSVN(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals accessing svn repositories