Update develop branch of cdeps to the most recent version of ESCOMP/CDEPS

cime_config/stream_cdeps.py

@@ -19,30 +19,37 @@
 
 _ymd_re = re.compile(r"%(?P<digits>[1-9][0-9]*)?y(?P<month>m(?P<day>d)?)?")
 
-_stream_nuopc_file_template = """
+_stream_file_template = """
   <stream_info name="{streamname}">
    <taxmode>{stream_taxmode}</taxmode>
-   <tInterpAlgo>{stream_tintalgo}</tInterpAlgo>
-   <readMode>{stream_readmode}</readMode>
+   <tintalgo>{stream_tintalgo}</tintalgo>
+   <readmode>{stream_readmode}</readmode>
    <mapalgo>{stream_mapalgo}</mapalgo>
    <dtlimit>{stream_dtlimit}</dtlimit>
-   <yearFirst>{stream_year_first}</yearFirst>
-   <yearLast>{stream_year_last}</yearLast>
-   <yearAlign>{stream_year_align}</yearAlign>
-   <stream_vectors>{stream_vectors}</stream_vectors>
-   <stream_mesh_file>{stream_meshfile}</stream_mesh_file>
-   <stream_lev_dimname>{stream_lev_dimname}</stream_lev_dimname>
-   <stream_data_files>
+   <year_first>{stream_year_first}</year_first>
+   <year_last>{stream_year_last}</year_last>
+   <year_align>{stream_year_align}</year_align>
+   <vectors>{stream_vectors}</vectors>
+   <meshfile>{stream_meshfile}</meshfile>
+   <lev_dimname>{stream_lev_dimname}</lev_dimname>
+   <datafiles>
       {stream_datafiles}
-   </stream_data_files>
-   <stream_data_variables>
+   </datafiles>
+   <datavars>
       {stream_datavars}
-   </stream_data_variables>
-   <stream_offset>{stream_offset}</stream_offset>
- </stream_info>
+   </datavars>
+   <offset>{stream_offset}</offset>
+  </stream_info>
 
 """
 
+valid_values = {}
+valid_values["mapalgo"]  = ["bilinear", "nn", "redist", "mapconsd", "mapconf"]
+valid_values["tintalgo"] = ["lower", "upper", "nearest", "linear", "coszen"]
+valid_values["taxmode"]  = ["cycle", "extend", "limit"]
+
+xml_scalar_names = ["stream_meshfile", "stream_mapalgo", "stream_tintalgo", "stream_taxmode", "stream_dtlimit"]
+
 class StreamCDEPS(GenericXML):
 
     def __init__(self, infile, schema):
@@ -54,16 +61,77 @@ def __init__(self, infile, schema):
         if os.path.exists(infile):
             GenericXML.read(self, infile, schema)
 
-    def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file, available_neon_data=None):
+    def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file, user_mods_file,
+                          available_neon_data=None):
         """
         Create the stream xml file and append the required stream input data to the input data list file
         available_neon_data is an optional list of NEON tower data available for the given case, if provided
         this data will be used to populate the NEON streamdata list
         """
+
+        # determine if there are user mods
+        lines_input = []
+        expect (os.path.isfile(user_mods_file),
+                "No file {} found in case directory".format(user_mods_file))
+        with open(user_mods_file, "r", encoding='utf-8') as stream_mods_file:
+            lines_input = stream_mods_file.readlines()
+        stream_mod_dict = {}
+        n = len(lines_input)
+
+        index = 0
+        lines_input_new = []
+        while index < len(lines_input):
+            line = lines_input[index].strip()
+            if line.startswith('!') or (not line):
+                index = index + 1
+                continue
+            while line[-1] == '\\':
+                index += 1
+                if index < len(lines_input):
+                    line = line[:-1].strip() + ' ' + lines_input[index].strip()
+                else:
+                    line = line.replace('\\', '').strip()
+                    break
+                # endif
+            # end while
+            index += 1 
+            lines_input_new.append(line)
+        #end while
+
+        for line in lines_input_new:
+            # read in a single line in user_nl_xxx_streams and parse it if it is not a comment
+            stream_mods = [x.strip() for x in line.strip().split(":") if x]
+            expect(len(stream_mods) == 2,
+                   "input stream mod can only be of the form streamname:var=value(s)")
+            stream,varmod = stream_mods
+            expect (stream in stream_names, 
+                    "{} contains a streamname \'{}\' that is not part of valid streamnames {}".
+                    format(user_mods_file,stream,stream_names)) 
+            if stream  not in stream_mod_dict:
+                stream_mod_dict[stream] = {}
+            # var=value and check the validity
+            varmod_args = [x.strip() for x in varmod.split("=") if x]
+            expect(len(varmod_args) == 2,
+                   "input stream mod can only be of the form streamname:var=value(s)")
+            # allow multiple entries for varmod_args, most recent wins
+            varname,varval = varmod_args
+            if varname in stream_mod_dict[stream]:
+                logger.warning("varname {} is already in stream mod dictionary".format(varname))
+
+            if varname == "datavars" or varname == "datafiles":
+                if varname == "datavars":
+                    varvals = ["<var>{}</var>".format(x.strip()) for x in varval.split(",") if x]
+                if varname == "datafiles":
+                    varvals = ["<file>{}</file>".format(x.strip()) for x in varval.split(",") if x]
+                varval = "      " + "\n      ".join(varvals)
+                varval = varval.strip()
+            stream_mod_dict[stream][varname] = varval
+
         # write header of stream file
-        with open(streams_xml_file, 'w') as stream_file:
+        with open(streams_xml_file, 'w', encoding='utf-8') as stream_file:
             stream_file.write('<?xml version="1.0"?>\n')
             stream_file.write('<file id="stream" version="2.0">\n')
+
         # write contents of stream file
         for stream_name in stream_names:
             if stream_name.startswith("NEON."):
@@ -131,11 +199,8 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file
                             stream_vars[node_name] += "\n      " + self._add_xml_delimiter(stream_datafiles.split("\n"), "file")
                         else:
                             stream_vars[node_name] = self._add_xml_delimiter(stream_datafiles.split("\n"), "file")
-                elif (   node_name == 'stream_meshfile'
-                      or node_name == 'stream_mapalgo'
-                      or node_name == 'stream_tintalgo'
-                      or node_name == 'stream_taxmode'
-                      or node_name == 'stream_dtlimit'):
+                        #endif
+                elif (node_name in xml_scalar_names):
                     attributes['model_grid'] = case.get_value("GRID")
                     attributes['compset'] = case.get_value("COMPSET")
                     value = self._get_value_match(node, node_name[7:], attributes=attributes)
@@ -148,18 +213,34 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file
                     # Get the other dependencies
                     stream_dict = self._add_value_to_dict(stream_vars, case, node)
 
+            # substitute user_mods in generated stream file (i.e. stream_vars)
+            mod_dict = {}
+            if stream_vars['streamname'] in stream_mod_dict:
+                mod_dict = stream_mod_dict[stream_vars['streamname']]
+                for var_key in mod_dict:
+                    expect( 'stream_' + var_key in stream_vars,
+                            "stream mod {} is not a valid name in {}".format(var_key,user_mods_file))
+                    if var_key in valid_values:
+                        expect(mod_dict[var_key] in valid_values[var_key],
+                               "{} can only have values of {} for stream {} in file {}".
+                               format(var_key, valid_values[var_key], stream_name, user_mods_file))
+                    stream_vars['stream_' + var_key] = mod_dict[var_key]
+                    if var_key == 'datafiles':
+                        stream_datafiles = mod_dict[var_key]
+                        stream_datafiles = stream_datafiles.replace('<file>','').replace('</file>','')
+
             # append to stream xml file
-            stream_file_text = _stream_nuopc_file_template.format(**stream_vars)
-            with open(streams_xml_file, 'a') as stream_file:
-                stream_file.write(stream_file_text)
+            stream_file_text = _stream_file_template.format(**stream_vars)
+            with open(streams_xml_file, 'a', encoding='utf-8') as stream_file:
+                stream_file.write(case.get_resolved_value(stream_file_text))
 
             # append to input_data_list
             if stream_vars['stream_meshfile']:
                 stream_meshfile = stream_vars['stream_meshfile'].strip()
                 self._add_entries_to_inputdata_list(stream_meshfile, stream_datafiles.split("\n"), data_list_file)
 
         # write close of stream xml file
-        with open(streams_xml_file, 'a') as stream_file:
+        with open(streams_xml_file, 'a', encoding='utf-8') as stream_file:
             stream_file.write("</file>\n")
 
     def _get_stream_first_and_last_dates(self, stream, case):
@@ -181,7 +262,7 @@ def _add_entries_to_inputdata_list(self, stream_meshfile, stream_datafiles, data
         and writes out the new file
         """
         lines_hash = self._get_input_file_hash(data_list_file)
-        with open(data_list_file, 'a') as input_data_list:
+        with open(data_list_file, 'a', encoding='utf-8') as input_data_list:
             # write out the mesh file separately
             string = "mesh = {}\n".format(stream_meshfile)
             hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
@@ -191,7 +272,7 @@ def _add_entries_to_inputdata_list(self, stream_meshfile, stream_datafiles, data
             for i, filename in enumerate(stream_datafiles):
                 if filename.strip() == '':
                     continue
-                string = "file{:d} = {}\n".format(i+1, filename)
+                string = "file{:d} = {}\n".format(i+1, filename.strip())
                 hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
                 if hashValue not in lines_hash:
                     input_data_list.write(string)
@@ -202,7 +283,7 @@ def _get_input_file_hash(self, data_list_file):
         """
         lines_hash = set()
         if os.path.isfile(data_list_file):
-            with open(data_list_file, "r") as input_data_list:
+            with open(data_list_file, "r", encoding='utf-8') as input_data_list:
                 for line in input_data_list:
                     hashValue = hashlib.md5(line.rstrip().encode('utf-8')).hexdigest()
                     logger.debug( "Found line {} with hash {}".format(line,hashValue))

cime_config/streams.xsd

@@ -12,33 +12,33 @@
   <!-- simple elements -->
   <!-- ======================== -->
 
-  <xs:element name="taxmode" type="xs:string"/>
-  <xs:element name="tInterpAlgo"  type="xs:string"/>
-  <xs:element name="readMode"   type="xs:string"/>
-  <xs:element name="mapalgo"  type="xs:string"/>
-  <xs:element name="dtlimit"      type="xs:double"/>
-  <xs:element name="yearFirst" type="xs:integer"/>
-  <xs:element name="yearLast" type="xs:integer"/>
-  <xs:element name="yearAlign" type="xs:integer"/>
-  <xs:element name="stream_offset" type="xs:integer"/>
-  <xs:element name="stream_vectors" type="xs:string"/>
-  <xs:element name="stream_mesh_file" type="xs:anyURI"/>
-  <xs:element name="stream_lev_dimname" type="xs:string"/>
-  <xs:element name="var" type="xs:string"/>
+  <xs:element name="taxmode"     type="xs:string"/>
+  <xs:element name="tintalgo"    type="xs:string"/>
+  <xs:element name="readmode"    type="xs:string"/>
+  <xs:element name="mapalgo"     type="xs:string"/>
+  <xs:element name="dtlimit"     type="xs:double"/>
+  <xs:element name="year_first"  type="xs:integer"/>
+  <xs:element name="year_last"   type="xs:integer"/>
+  <xs:element name="year_align"  type="xs:integer"/>
+  <xs:element name="offset"      type="xs:integer"/>
+  <xs:element name="vectors"     type="xs:string"/>
+  <xs:element name="meshfile"    type="xs:anyURI"/>
+  <xs:element name="lev_dimname" type="xs:string"/>
+  <xs:element name="var"         type="xs:string"/>
 
   <!-- ======================== -->
   <!-- complex elements -->
   <!-- ======================== -->
 
-  <xs:element name="stream_data_files">
+  <xs:element name="datafiles">
     <xs:complexType>
       <xs:sequence>
         <xs:element name="file" type="xs:anyURI"  minOccurs="1" maxOccurs="unbounded"/>
       </xs:sequence>
     </xs:complexType>
   </xs:element>
 
-  <xs:element name="stream_data_variables">
+  <xs:element name="datavars">
     <xs:complexType>
       <xs:sequence>
         <xs:element ref="var" minOccurs="1" maxOccurs="unbounded"/>
@@ -49,20 +49,20 @@
   <xs:element name="stream_info">
     <xs:complexType>
       <xs:sequence>
-        <xs:element ref="taxmode"/>
-        <xs:element ref="tInterpAlgo" />
-        <xs:element ref="readMode" />
-        <xs:element ref="mapalgo" />
-        <xs:element ref="dtlimit" />
-        <xs:element ref="yearFirst" />
-        <xs:element ref="yearLast" />
-        <xs:element ref="yearAlign" />
-        <xs:element ref="stream_vectors" />
-        <xs:element ref="stream_mesh_file" />
-        <xs:element ref="stream_lev_dimname"/>
-        <xs:element ref="stream_data_files" />
-        <xs:element ref="stream_data_variables" />
-        <xs:element ref="stream_offset" />
+        <xs:element ref="taxmode"     />
+        <xs:element ref="tintalgo"    />
+        <xs:element ref="readmode"    />
+        <xs:element ref="mapalgo"     />
+        <xs:element ref="dtlimit"     />
+        <xs:element ref="year_first"   />
+        <xs:element ref="year_last"    />
+        <xs:element ref="year_align"   />
+        <xs:element ref="vectors"     />
+        <xs:element ref="meshfile"    />
+        <xs:element ref="lev_dimname" />
+        <xs:element ref="datafiles"   />
+        <xs:element ref="datavars"    />
+        <xs:element ref="offset"      />
       </xs:sequence>
       <xs:attribute ref="name" />
     </xs:complexType>

datm/atm_comp_nuopc.F90

@@ -34,7 +34,7 @@ module cdeps_datm_comp
   use dshr_strdata_mod , only : shr_strdata_get_stream_pointer, shr_strdata_setOrbs
   use dshr_mod         , only : dshr_model_initphase, dshr_init
   use dshr_mod         , only : dshr_state_setscalar, dshr_set_runclock, dshr_log_clock_advance
-  use dshr_mod         , only : dshr_restart_read, dshr_restart_write, dshr_mesh_init
+  use dshr_mod         , only : dshr_mesh_init, dshr_check_restart_alarm
   use dshr_mod         , only : dshr_orbital_init, dshr_orbital_update
   use dshr_dfield_mod  , only : dfield_type, dshr_dfield_add, dshr_dfield_copy
   use dshr_fldlist_mod , only : fldlist_type, dshr_fldlist_add, dshr_fldlist_realize
@@ -448,7 +448,7 @@ subroutine ModelAdvance(gcomp, rc)
     type(ESMF_Time)         :: nextTime
     type(ESMF_TimeInterval) :: timeStep
     real(r8)                :: nextsw_cday
-    logical                 :: restart_write ! restart alarm is ringing
+    logical                 :: restart_write         ! restart alarm is ringing
     integer                 :: next_ymd      ! model date
     integer                 :: next_tod      ! model sec into model date
     integer                 :: yr, mon, day  ! year, month, day
@@ -485,18 +485,9 @@ subroutine ModelAdvance(gcomp, rc)
          orbEccen, orbObliqr, orbLambm0, orbMvelpp, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
-    ! determine if will write restart
-    call ESMF_ClockGetAlarm(clock, alarmname='alarm_restart', alarm=alarm, rc=rc)
+    restart_write = dshr_check_restart_alarm(clock, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
-    if (ESMF_AlarmIsRinging(alarm, rc=rc)) then
-       if (ChkErr(rc,__LINE__,u_FILE_u)) return
-       call ESMF_AlarmRingerOff( alarm, rc=rc )
-       if (ChkErr(rc,__LINE__,u_FILE_u)) return
-       restart_write = .true.
-    else
-       restart_write = .false.
-    endif
-
+
     ! Run datm
     call ESMF_TraceRegionEnter('datm_run')
     call datm_comp_run(importstate, exportstate, next_ymd, next_tod, mon, &