Introduce multi instance capability

cime_config/buildlib

@@ -62,6 +62,7 @@ def buildlib(caseroot, libroot, bldroot):
         # create Filepath file for mom
         #-------------------------------------------------------
         memory_mode =  case.get_value("MOM6_MEMORY_MODE")
+        infra_api =  case.get_value("MOM6_INFRA_API")
 
         user_incldir = "\"-I{} -I{} -I{}\"".\
                         format(os.path.join(srcroot,"libraries","FMS","src","include"),
@@ -75,7 +76,7 @@ def buildlib(caseroot, libroot, bldroot):
             paths = [os.path.join(caseroot,"SourceMods","src.mom"),
                      os.path.join(comp_root_dir_ocn,"MOM6","config_src","drivers", driver),
                      os.path.join(comp_root_dir_ocn,"MOM6","config_src","memory", memory_mode),
-                     os.path.join(comp_root_dir_ocn,"MOM6","config_src","infra", "FMS1"),
+                     os.path.join(comp_root_dir_ocn,"MOM6","config_src","infra", infra_api),
                      os.path.join(comp_root_dir_ocn,"MOM6","config_src","external","GFDL_ocean_BGC"),
                      os.path.join(comp_root_dir_ocn,"MOM6","config_src","external","ODA_hooks"),
                      os.path.join(comp_root_dir_ocn,"MOM6","config_src","external","stochastic_physics"),

cime_config/buildnml

@@ -9,7 +9,7 @@
 # Disable these because this is our standard setup
 # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position
 
-import os, shutil, sys
+import os, shutil, sys, re
 
 CIMEROOT = os.environ.get("CIMEROOT")
 if CIMEROOT is None:
@@ -23,15 +23,14 @@ sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"MOM_RPS
 from standard_script_setup import *
 from CIME.case import Case
 from CIME.buildnml import parse_input
-from CIME.utils import run_cmd
 from FType_MOM_params import FType_MOM_params
 from FType_input_nml import FType_input_nml
 from FType_input_data_list import FType_input_data_list
 from FType_diag_table import FType_diag_table
 
 logger = logging.getLogger(__name__)
 
-def prep_input(case):
+def prep_input(case, inst_suffixes):
     """ Generates out-of-the-box versions of MOM6 input files including MOM_input, MOM_override, diag_table
     input.nml, and mom.input_data_list, inside the run directory. If any of these input files are provided
     in SourceMods, those versions will be copied to run directory instead."""
@@ -41,6 +40,8 @@ def prep_input(case):
     comp_root_dir_ocn = case.get_value("COMP_ROOT_DIR_OCN")
     caseroot = case.get_value("CASEROOT")
     SourceMods_dir = os.path.join(caseroot,"SourceMods","src.mom")
+    SourceMods_listdir = os.listdir(SourceMods_dir)
+    multi_instance = inst_suffixes[0] != ''
 
     # Make sure that rundir exists. If not, make it:
     if not os.path.exists(rundir):
@@ -50,93 +51,109 @@ def prep_input(case):
     json_templates_dir = os.path.join(comp_root_dir_ocn,"param_templates","json")
 
     # 1. Create MOM_input:
-    if "MOM_input" in os.listdir(SourceMods_dir):
-        run_cmd("cp "+os.path.join(SourceMods_dir,"MOM_input")+" "+rundir)
+    MOM_input_template = os.path.join(json_templates_dir, "MOM_input.json")
+    MOM_input_rundir = os.path.join(rundir,f"MOM_input{inst_suffixes[0]}")
+    if multi_instance:
+        # don't allow separate MOM_input files for separate instances
+        assert not any([re.match("MOM_input_+\d", filename) for filename in SourceMods_listdir]), \
+            "Cannot provide separate instances of MOM_input"
+    if "MOM_input" in SourceMods_listdir:
+        shutil.copy(os.path.join(SourceMods_dir,"MOM_input"), MOM_input_rundir)
     else:
-        MOM_input_src = os.path.join(json_templates_dir, "MOM_input.json")
-        MOM_input_dest = os.path.join(rundir,"MOM_input")
-        MOM_input = FType_MOM_params.from_json(MOM_input_src)
-        MOM_input.write(output_path=MOM_input_dest, output_format="MOM_input", case=case)
+        # Create MOM_input in rundir using template
+        MOM_input = FType_MOM_params.from_json(MOM_input_template)
+        MOM_input.write(output_path=MOM_input_rundir, output_format="MOM_input", case=case)
+    # If multi-instance, create MOM_input copies for each instance
+    for inst_suffix in inst_suffixes[1:]:
+        shutil.copy(MOM_input_rundir, os.path.join(rundir,f"MOM_input{inst_suffix}"))
+
 
     # 2. Create MOM_override:
-    user_nl_mom = FType_MOM_params.from_MOM_input(os.path.join(caseroot,"user_nl_mom"))
-    if "MOM_override" in os.listdir(SourceMods_dir):
-        assert len(user_nl_mom.data)==0, "Cannot provide parameter changes via both SourceMods and user_nl_mom!"
-        run_cmd("cp "+os.path.join(SourceMods_dir,"MOM_override")+" "+rundir)
-    else:
-        init_MOM_override(rundir)
-        process_user_nl_mom(case)
+    for inst_suffix in inst_suffixes:
+        user_nl_mom = FType_MOM_params.from_MOM_input(os.path.join(caseroot,f"user_nl_mom{inst_suffix}"))
+        if f"MOM_override{inst_suffix}" in SourceMods_listdir:
+            assert len(user_nl_mom.data)==0, "Cannot provide parameter changes via both SourceMods and user_nl_mom!"
+            shutil.copy(os.path.join(SourceMods_dir,f"MOM_override{inst_suffix}"),
+                        os.path.join(rundir,f"MOM_override{inst_suffix}"))
+        else:
+            init_MOM_override(rundir, inst_suffix)
+            process_user_nl_mom(case, inst_suffix)
 
     # 3. Read in final versions of MOM_input and MOM_override, so as to use them when inferring
-    #    values of expandable variables in the templates of below MOM6 input files.
-    MOM_input_final = FType_MOM_params.from_MOM_input(os.path.join(rundir,"MOM_input"))
-    MOM_override_final = FType_MOM_params.from_MOM_input(os.path.join(rundir,"MOM_override"))
+    #    values of expandable variables in the templates of subsequent MOM6 input files.
+    MOM_input_final = FType_MOM_params.from_MOM_input(MOM_input_rundir)
+    MOM_override_final = FType_MOM_params.from_MOM_input(os.path.join(rundir,f"MOM_override{inst_suffixes[0]}"))
     MOM_input_final.append(MOM_override_final)
 
     # 4. Create input.nml:
-    if "input.nml" in os.listdir(SourceMods_dir):
-        run_cmd("cp "+os.path.join(SourceMods_dir,"input.nml")+" "+rundir)
+    input_nml_template = os.path.join(json_templates_dir, "input_nml.json")
+    input_nml_srcmod = os.path.join(SourceMods_dir,"input.nml") 
+    input_nml_rundir = os.path.join(rundir,"input.nml") 
+    if "input.nml" in SourceMods_listdir:
+        shutil.copy(input_nml_srcmod, input_nml_rundir)
     else:
-        input_nml_src = os.path.join(json_templates_dir, "input_nml.json")
-        input_nml_dest = os.path.join(rundir,"input.nml")
-        input_nml = FType_input_nml.from_json(input_nml_src)
-        input_nml.write(input_nml_dest, case)
+        input_nml = FType_input_nml.from_json(input_nml_template)
+        input_nml.write(input_nml_rundir, case)
 
     # 5. Create mom.input_data_list:
-    input_data_list_src = os.path.join(json_templates_dir, "input_data_list.json")
-    input_data_list_dest = os.path.join(Buildconf,"mom.input_data_list")
-    input_data_list = FType_input_data_list.from_json(input_data_list_src)
-    input_data_list.write(input_data_list_dest, case, MOM_input_final)
+    input_data_list_template = os.path.join(json_templates_dir, "input_data_list.json")
+    input_data_list_buildconf = os.path.join(Buildconf,"mom.input_data_list")
+    input_data_list = FType_input_data_list.from_json(input_data_list_template)
+    input_data_list.write(input_data_list_buildconf, case, MOM_input_final)
 
     # 6. Create diag_table:
-    if "diag_table" in os.listdir(SourceMods_dir):
-        run_cmd("cp "+os.path.join(SourceMods_dir,"diag_table")+" "+rundir)
+    diag_table_template = os.path.join(json_templates_dir, "diag_table.json")
+    diag_table_srcmod = os.path.join(SourceMods_dir, "diag_table")
+    diag_table_rundir = os.path.join(rundir,"diag_table")
+    if "diag_table" in SourceMods_listdir:
+        shutil.copy(diag_table_srcmod, diag_table_rundir)
     else:
-        diag_table_src = os.path.join(json_templates_dir, "diag_table.json")
-        diag_table_dest = os.path.join(rundir,"diag_table")
-        diag_table = FType_diag_table.from_json(diag_table_src)
-        diag_table.write(diag_table_dest, case, MOM_input_final)
+        diag_table = FType_diag_table.from_json(diag_table_template)
+        diag_table.write(diag_table_rundir, case, MOM_input_final)
 
-def init_MOM_override(rundir):
+def init_MOM_override(rundir, inst_suffix):
     # Create an empty MOM_override:
-    with open(os.path.join(rundir,"MOM_override"), 'w') as MOM_override:
+    with open(os.path.join(rundir,f"MOM_override{inst_suffix}"), 'w') as MOM_override:
         MOM_override.write(\
             '! WARNING: DO NOT EDIT this file! Any user change made in this file will be\n'+\
             '!          overriden. This file is automatically generated. MOM6 parameter\n'+\
             '!          changes may be made via SourceMods or user_nl_mom.\n'+\
             '!-------------------------------------------------------------------------\n\n')
 
-def process_user_nl_mom(case):
+def process_user_nl_mom(case, inst_suffix):
     """ Calls the appropriate MOM_RPS functions to parse user_nl_mom and create MOM_override."""
     caseroot = case.get_value("CASEROOT")
     rundir   = case.get_value("RUNDIR")
 
-    user_nl_mom = FType_MOM_params.from_MOM_input(os.path.join(caseroot,"user_nl_mom"))
+    user_nl_mom = FType_MOM_params.from_MOM_input(os.path.join(caseroot,f"user_nl_mom{inst_suffix}"))
 
     # copy the user_nl_mom parameters into MOM_override:
     if len(user_nl_mom.data)>0:
 
         # check if a copy of MOM_override is provided in SourceMods:
         SourceMods_dir = os.path.join(caseroot,"SourceMods","src.mom")
-        if "MOM_override" in os.listdir(SourceMods_dir):
+        if f"MOM_override{inst_suffix}" in os.listdir(SourceMods_dir):
             raise SystemExit("ERROR: Cannot provide parameter changes via both SourceMods and user_nl_mom!")
 
         # parse the MOM_input file staged in rundir:
-        MOM_input_rundir = FType_MOM_params.from_MOM_input(os.path.join(rundir,"MOM_input"))
+        MOM_input_rundir = FType_MOM_params.from_MOM_input(os.path.join(rundir,f"MOM_input{inst_suffix}"))
 
         # Write MOM_override (based on data from user_nl_mom)
-        user_nl_mom.write(output_path = os.path.join(rundir,"MOM_override"),
+        user_nl_mom.write(output_path = os.path.join(rundir,f"MOM_override{inst_suffix}"),
                           output_format = "MOM_override",
                           def_params = MOM_input_rundir)
 
-def _copy_input_files(case, dest_dir):
+def _copy_input_files(case, dest_dir, inst_suffixes):
     """ Saves copies of MOM6 input files in momconf directory for the record."""
     caseroot = case.get_value("CASEROOT")
     rundir   = case.get_value("RUNDIR")
     if not os.path.isdir(dest_dir):
         os.makedirs(dest_dir)
 
-    for filename in ["MOM_input", "MOM_override", "diag_table", "input.nml"]:
+    for inst_suffix in inst_suffixes:
+        for filename in ["MOM_input", "MOM_override"]:
+            shutil.copy(os.path.join(rundir,filename+inst_suffix), dest_dir)
+    for filename in ["diag_table", "input.nml"]:
         shutil.copy(os.path.join(rundir,filename), dest_dir)
 
 
@@ -150,17 +167,20 @@ def buildnml(case, caseroot, compname):
     if compname != "mom":
         raise AttributeError
 
+    ninst = case.get_value("NINST_OCN")
+    inst_suffixes = ["_{:04d}".format(i+1) for i in range(ninst)] if ninst>1 else ['']
+
     # prepare all input files
-    prep_input(case)
+    prep_input(case, inst_suffixes)
 
     # save copies of input files in momconf
     caseroot = case.get_value("CASEROOT")
     momconfdir  = os.path.join(caseroot, "Buildconf", "momconf")
-    _copy_input_files(case, momconfdir)
+    _copy_input_files(case, momconfdir, inst_suffixes)
 
     # save copies of input files in CaseDocs
     casedocsdir  = os.path.join(caseroot, "CaseDocs")
-    _copy_input_files(case, casedocsdir)
+    _copy_input_files(case, casedocsdir, inst_suffixes)
 
     return
 

cime_config/config_archive.xml

@@ -24,7 +24,7 @@
     <hist_file_extension>Pacific_undercurrent.?[_\d+]+.nc</hist_file_extension>
     <hist_file_extension>Taiwan_Luzon.?[_\d+]+.nc</hist_file_extension>
     <hist_file_extension>Windward_Passage.?[_\d+]+.nc</hist_file_extension>
-    <hist_file_extension>static.nc$</hist_file_extension>
+    <hist_file_extension>static.?[_\d+]+.nc$</hist_file_extension>
     <rpointer>
       <rpointer_file>rpointer.ocn$NINST_STRING</rpointer_file>
     </rpointer>

cime_config/config_component.xml

@@ -30,6 +30,17 @@
     </desc>
   </entry>
 
+
+  <entry id="MOM6_INFRA_API">
+    <type>char</type>
+    <valid_values>FMS1,FMS2</valid_values>
+    <default_value>FMS2</default_value>
+    <group>build_component_mom</group>
+    <file>env_build.xml</file>
+    <desc> This variable controls the MOM6 infrastructure API.
+    </desc>
+  </entry>
+
   <entry id="OCN_DIAG_MODE">
     <type>char</type>
     <valid_values>spinup,production,development,none</valid_values>

param_templates/MOM_input.yaml

@@ -404,7 +404,7 @@ Global:
             "Name of the file from which to read horizontal grid data."
         datatype: string
         value:
-            $OCN_GRID == "gx1v6": "ocean_hgrid.nc"
+            $OCN_GRID == "gx1v6": "ocean_hgrid_230424.nc"
             $OCN_GRID == "tx0.66v1": "ocean_hgrid_180829.nc"
             $OCN_GRID == "tx0.25v1": "ocean_hgrid.nc"
     USE_TRIPOLAR_GEOLONB_BUG:
@@ -446,7 +446,7 @@ Global:
             The file from which the bathymetry is read."
         datatype: string
         value:
-            $OCN_GRID == "gx1v6": "ocean_topog.nc"
+            $OCN_GRID == "gx1v6": "ocean_topog_230424.nc"
             $OCN_GRID == "tx0.66v1": "ocean_topog_200701.nc"
             $OCN_GRID == "tx0.25v1": "ocean_topog.nc"
     MAXIMUM_DEPTH:
@@ -3361,6 +3361,14 @@ Global:
         datatype: string
         value:
             $COMP_WAV in ["ww3", "ww3dev"] and $WAV_GRID == "wtx0.66v1" : "0.04, 0.11, 0.33"
+    IO_LAYOUT:
+        description: |
+            The processor layout to be used, or 0,0 to automatically set the io_layout to
+            be the same as the layout.
+        datatype: list
+        value:
+            $OCN_GRID == "tx0.25v1":
+              4, 3
 
 
 KPP:

param_templates/input_nml.yaml

@@ -9,7 +9,11 @@ MOM_input_nml:
     restart_output_dir:
         values: "'./'"
     parameter_filename:
-        values: "'MOM_input', 'MOM_override'"
+        values:
+          $NINST_OCN > 1:
+            "'MOM_input_%4E', 'MOM_override_%4E'"
+          else:
+            "'MOM_input', 'MOM_override'"
 fms_nml:
     clock_grain:
         values: "'ROUTINE'"

param_templates/json/MOM_input.json

@@ -46,9 +46,11 @@
          }
       },
       "IO_LAYOUT": {
-         "description": "\"default = 0\nThe processor layout to be used, or 0,0 to automatically\nset the io_layout to be the same as the layout.\"\n",
-         "datatype": "string",
-         "value": "1, 1"
+         "description": "The processor layout to be used, or 0,0 to automatically set the io_layout to\nbe the same as the layout.\n",
+         "datatype": "list",
+         "value": {
+            "$OCN_GRID == \"tx0.25v1\"": "4, 3"
+         }
       },
       "NK": {
          "description": "\"[nondim]\nThe number of model layers.\"\n",
@@ -281,7 +283,7 @@
          "description": "\"Name of the file from which to read horizontal grid data.\"\n",
          "datatype": "string",
          "value": {
-            "$OCN_GRID == \"gx1v6\"": "ocean_hgrid.nc",
+            "$OCN_GRID == \"gx1v6\"": "ocean_hgrid_230424.nc",
             "$OCN_GRID == \"tx0.66v1\"": "ocean_hgrid_180829.nc",
             "$OCN_GRID == \"tx0.25v1\"": "ocean_hgrid.nc"
          }
@@ -307,7 +309,7 @@
          "description": "\"default = 'topog.nc'\nThe file from which the bathymetry is read.\"\n",
          "datatype": "string",
          "value": {
-            "$OCN_GRID == \"gx1v6\"": "ocean_topog.nc",
+            "$OCN_GRID == \"gx1v6\"": "ocean_topog_230424.nc",
             "$OCN_GRID == \"tx0.66v1\"": "ocean_topog_200701.nc",
             "$OCN_GRID == \"tx0.25v1\"": "ocean_topog.nc"
          }

param_templates/json/input_nml.json

@@ -10,7 +10,10 @@
          "values": "'./'"
       },
       "parameter_filename": {
-         "values": "'MOM_input', 'MOM_override'"
+         "values": {
+            "$NINST_OCN > 1": "'MOM_input_%4E', 'MOM_override_%4E'",
+            "else": "'MOM_input', 'MOM_override'"
+         }
       }
    },
    "fms_nml": {