From 5693a83e49d20c6bf8b4a2e6682886ad89333d9b Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Wed, 20 Jan 2021 10:14:19 -0700 Subject: [PATCH 01/42] bug fixes for aux_cam test suite --- cime_config/buildnml | 4 ++-- mediator/med_io_mod.F90 | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index ea6827d05..38c709973 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -258,13 +258,13 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # skip the mediator if there is a prognostic component and all other components are stub valid_comps.remove("CPL") nmlgen.set_value('mediator_present', value='.false.') - nmlgen.set_value("drv_restart_pointer", value="none") nmlgen.set_value("component_list", value=" ".join(valid_comps)) else: # do not skip mediator if there is a data component but all other components are stub - nmlgen.set_value("drv_restart_pointer", value="rpointer.cpl") valid_comps_string = " ".join(valid_comps) nmlgen.set_value("component_list", value=valid_comps_string.replace("CPL","MED")) + # the driver restart pointer will look like a mediator is present even if it is not + nmlgen.set_value("drv_restart_pointer", value="rpointer.cpl") logger.info("Writing nuopc_runseq for components {}".format(valid_comps)) nuopc_config_file = os.path.join(confdir, "nuopc.runconfig") diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 0a4fa7753..de2b5fb80 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -457,7 +457,7 @@ subroutine med_io_write_FB(filename, iam, FB, whead, wdata, nx, ny, nt, & real(r8), pointer :: fldptr1(:) real(r8), pointer :: fldptr2(:,:) real(r8), allocatable :: ownedElemCoords(:), ownedElemCoords_x(:), ownedElemCoords_y(:) - character(len=number_strlen) :: cnumber + character(16) :: cnumber character(CL) :: tmpstr type(ESMF_Field) :: lfield integer :: rank @@ -617,6 +617,7 @@ subroutine med_io_write_FB(filename, iam, FB, whead, wdata, nx, ny, nt, & call ESMF_FieldGet(lfield, ungriddedUBound=ungriddedUBound, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return write(cnumber,'(i0)') ungriddedUbound(1) + write(6,'(a,i8,a)')'DEBUG: itemc,ungriddedUbound,cnumber= '//trim(itemc),ungriddedUbound(1),cnumber call ESMF_LogWrite(trim(subname)//':'//'field '//trim(itemc)// & ' has an griddedUBound of '//trim(cnumber), ESMF_LOGMSG_INFO) From e8f967eb11bbed8387a2f81cd8249b9dbf0288b9 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 26 Jan 2021 19:15:54 -0700 Subject: [PATCH 02/42] changes to get aux_cam test suite working --- cime_config/config_component_cesm.xml | 69 ++++++++++++++----------- cime_config/namelist_definition_drv.xml | 43 ++++++++++----- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/cime_config/config_component_cesm.xml b/cime_config/config_component_cesm.xml index 7c4c52e44..bf75e52ba 100644 --- a/cime_config/config_component_cesm.xml +++ b/cime_config/config_component_cesm.xml @@ -205,35 +205,46 @@ 48 - 24 - 24 - 24 - 24 - 24 - 24 - 48 - 48 - 1 - 96 - 96 - 96 - 96 - 192 - 192 - 192 - 192 - 384 - 384 - 384 - 144 - 72 - 144 - 288 - 48 - 48 - 24 - 24 - 1 + 24 + 24 + 24 + 24 + + + + + 24 + 24 + 48 + 48 + 1 + 96 + 96 + 96 + 96 + 192 + 192 + 192 + 192 + 384 + 384 + 384 + 384 + 192 + 384 + 48 + 48 + 48 + 96 + 192 + 72 + 144 + 288 + 48 + 48 + 24 + 24 + 1 diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 3ac72b245..24d22dda6 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -681,7 +681,7 @@ logical expdef - ALLCOMP_attributes + ATM_attributes Perpetual flag @@ -689,11 +689,10 @@ .false. - integer expdef - ALLCOMP_attributes + ATM_attributes Perpetual date @@ -746,7 +745,7 @@ logical expdef - ALLCOMP_attributes + ATM_attributes true => turn on aquaplanet mode in cam @@ -1076,39 +1075,55 @@ - + char mapping - ATM_attributes + ALLCOMP_attributes - MESH description of atm grid + DOMAIN file needed for single column model + A nearest neighbor search is done to match the PTS_LAT and PTS_LON + to the closest point in the domain file. This file is ONLY used in + single column mode. - $ATM_DOMAIN_MESH + $ATM_DOMAIN_PATH/$ATM_DOMAIN_FILE - + char mapping LND_attributes - MESH description of lnd grid + DOMAIN description of lnd grid - this will be depracated + only here for backwards compatibility - $LND_DOMAIN_MESH + $LND_DOMAIN_PATH/$LND_DOMAIN_FILE - + + char + mapping + ATM_attributes + + MESH description of atm grid + + + $ATM_DOMAIN_MESH + + + + char mapping LND_attributes - DOMAIN description of lnd grid + MESH description of lnd grid - $LND_DOMAIN_PATH/$LND_DOMAIN_FILE + $LND_DOMAIN_MESH From 2729830593ba18950a01a0ca4c4f590077891928 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Mon, 1 Feb 2021 09:12:48 -0700 Subject: [PATCH 03/42] fixed problems for cart3d mapping and made it the default for cesm --- cime_config/namelist_definition_drv.xml | 5 +- mediator/esmFldsExchange_cesm_mod.F90 | 6 +- mediator/med_io_mod.F90 | 1 - mediator/med_map_mod.F90 | 289 +++++++++++++----------- 4 files changed, 159 insertions(+), 142 deletions(-) diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 24d22dda6..5d146780b 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -1743,9 +1743,10 @@ logical mapping MED_attributes - used for atm->ocn and atm-ice mapping of u and v; rotate u,v to 3d cartesian space, map from src->dest, then rotate back + used for atm->ocn and atm-ice mapping of u and v; rotate u,v + to 3d cartesian space, map from src->dest, then rotate back - .false. + .true. diff --git a/mediator/esmFldsExchange_cesm_mod.F90 b/mediator/esmFldsExchange_cesm_mod.F90 index 3d082892e..8253fe951 100644 --- a/mediator/esmFldsExchange_cesm_mod.F90 +++ b/mediator/esmFldsExchange_cesm_mod.F90 @@ -1563,7 +1563,11 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc) if ( fldchk(is_local%wrap%FBexp(compice) , trim(fldname), rc=rc) .and. & fldchk(is_local%wrap%FBImp(compatm,compatm ), trim(fldname), rc=rc)) then if (trim(fldname) == 'Sa_u' .or. trim(fldname) == 'Sa_v') then - call addmap(fldListFr(compatm)%flds, trim(fldname), compice, mappatch, 'one', atm2ice_vmap) + if (mapuv_with_cart3d) then + call addmap(fldListFr(compatm)%flds, trim(fldname), compice, mappatch_uv3d, 'one', atm2ice_vmap) + else + call addmap(fldListFr(compatm)%flds, trim(fldname), compice, mappatch, 'one', atm2ice_vmap) + end if else call addmap(fldListFr(compatm)%flds, trim(fldname), compice, mapbilnr, 'one', atm2ice_smap) end if diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index de2b5fb80..60746534a 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -617,7 +617,6 @@ subroutine med_io_write_FB(filename, iam, FB, whead, wdata, nx, ny, nt, & call ESMF_FieldGet(lfield, ungriddedUBound=ungriddedUBound, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return write(cnumber,'(i0)') ungriddedUbound(1) - write(6,'(a,i8,a)')'DEBUG: itemc,ungriddedUbound,cnumber= '//trim(itemc),ungriddedUbound(1),cnumber call ESMF_LogWrite(trim(subname)//':'//'field '//trim(itemc)// & ' has an griddedUBound of '//trim(cnumber), ESMF_LOGMSG_INFO) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 2467d56a3..0c2970de5 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -179,9 +179,9 @@ subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapin character(len=*), parameter :: subname=' (module_MED_map:med_map_routehandles_initfrom_fieldbundle) ' !--------------------------------------------- - call t_startf('MED:'//subname) rc = ESMF_SUCCESS + call t_startf('MED:'//subname) if (dbug_flag > 1) then call ESMF_LogWrite(trim(subname)//": start", ESMF_LOGMSG_INFO) endif @@ -207,6 +207,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, use ESMF , only : ESMF_RouteHandle, ESMF_RouteHandlePrint, ESMF_Field, ESMF_MAXSTR use ESMF , only : ESMF_PoleMethod_Flag, ESMF_POLEMETHOD_ALLAVG use ESMF , only : ESMF_FieldSMMStore, ESMF_FieldRedistStore, ESMF_FieldRegridStore + use ESMF , only : ESMF_RouteHandleIsCreated use ESMF , only : ESMF_REGRIDMETHOD_BILINEAR, ESMF_REGRIDMETHOD_PATCH use ESMF , only : ESMF_REGRIDMETHOD_CONSERVE, ESMF_NORMTYPE_DSTAREA, ESMF_NORMTYPE_FRACAREA use ESMF , only : ESMF_UNMAPPEDACTION_IGNORE, ESMF_REGRIDMETHOD_NEAREST_STOD @@ -332,21 +333,27 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & + !DEBUG + ! TODO: CREATE flddst_debug to do this + ! dststatusfield=flddst_debug, & + !DEBUG rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return else if (mapindex == mappatch .or. mapindex == mappatch_uv3d) then - if (mastertask) then - write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) + if (.not. ESMF_RouteHandleIsCreated(routehandles(mappatch))) then + if (mastertask) then + write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) + end if + call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mappatch), & + srcMaskValues=(/srcMaskValue/), & + dstMaskValues=(/dstMaskValue/), & + regridmethod=ESMF_REGRIDMETHOD_PATCH, & + polemethod=polemethod, & + srcTermProcessing=srcTermProcessing_Value, & + ignoreDegenerate=.true., & + unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return end if - call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mappatch), & - srcMaskValues=(/srcMaskValue/), & - dstMaskValues=(/dstMaskValue/), & - regridmethod=ESMF_REGRIDMETHOD_PATCH, & - polemethod=polemethod, & - srcTermProcessing=srcTermProcessing_Value, & - ignoreDegenerate=.true., & - unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return else if (mastertask) then write(logunit,'(A)') trim(subname)//' mapindex '//trim(mapname)//' not supported for '//trim(string) @@ -812,146 +819,140 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d ! Loop over mapping types do mapindex = 1,nmappers - ! If packed field is created if (ESMF_FieldIsCreated(packed_data(mapindex)%field_src)) then - ! ----------------------------------- - ! Copy the src fields into the packed field bundle - ! ----------------------------------- - - call t_startf('MED:'//trim(subname)//' copy from src') - - ! First get the pointer for the packed source data - call ESMF_FieldGet(packed_data(mapindex)%field_src, farrayptr=dataptr2d_packed, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - - ! Now do the copy - do nf = 1,fieldcount - ! Get the indices into the packed data structure - np = packed_data(mapindex)%fldindex(nf) - if (np > 0) then - call ESMF_FieldGet(fieldlist_src(nf), ungriddedUBound=ungriddedUBound, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - if (ungriddedUBound(1) > 0) then - call ESMF_FieldGet(fieldlist_src(nf), farrayptr=dataptr2d, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - do nu = 1,ungriddedUBound(1) - dataptr2d_packed(np+nu-1,:) = dataptr2d(nu,:) - end do - else - call ESMF_FieldGet(fieldlist_src(nf), farrayptr=dataptr1d, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - dataptr2d_packed(np,:) = dataptr1d(:) - end if - end if - end do - call t_stopf('MED:'//trim(subname)//' copy from src') - - ! ----------------------------------- - ! Do the mapping - ! ----------------------------------- - - call t_startf('MED:'//trim(subname)//' map') - if (mapindex == mappatch_uv3d) then - ! for mappatch_uv3d do not use packed field bundles - call ESMF_FieldBundleGet(FBSrc, fieldName='Sa_u', field=usrc, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_FieldBundleGet(FBSrc, fieldName='Sa_v', field=vsrc, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_FieldBundleGet(FBDst, fieldName='Sa_u', field=udst, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_FieldBundleGet(FBDst, fieldName='Sa_v', field=vdst, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call med_map_uv_cart3d(usrc, vsrc, udst, vdst, routehandles, mappatch, rc=rc) + ! For mappatch_uv3d do not use packed field bundles + call med_map_uv_cart3d(FBsrc, FBdst, routehandles, mappatch, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - else if (mapindex == mapfcopy) then - - ! Mapping is redistribution - call ESMF_FieldRedist(& - packed_data(mapindex)%field_src, & - packed_data(mapindex)%field_dst, & - routehandles(mapindex), rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return + else - else if ( trim(packed_data(mapindex)%mapnorm) /= 'unset' .and. & - trim(packed_data(mapindex)%mapnorm) /= 'one' .and. & - trim(packed_data(mapindex)%mapnorm) /= 'none') then + ! ----------------------------------- + ! Copy the src fields into the packed field bundle + ! ----------------------------------- + + call t_startf('MED:'//trim(subname)//' copy from src') - ! Normalized mapping - assume that each packed field has only one normalization type - call ESMF_FieldBundleGet(FBFracSrc, packed_data(mapindex)%mapnorm, field=field_fracsrc, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call med_map_field_normalized(& - field_src=packed_data(mapindex)%field_src, & - field_dst=packed_data(mapindex)%field_dst, & - routehandles=routehandles, & - maptype=mapindex, & - field_normsrc=field_fracsrc, & - field_normdst=packed_data(mapindex)%field_fracdst, rc=rc) - - else if ( trim(packed_data(mapindex)%mapnorm) == 'one' .or. trim(packed_data(mapindex)%mapnorm) == 'none') then - - ! Mapping with no normalization that is not redistribution - call med_map_field (& - field_src=packed_data(mapindex)%field_src, & - field_dst=packed_data(mapindex)%field_dst, & - routehandles=routehandles, & - maptype=mapindex, rc=rc) + ! First get the pointer for the packed source data + call ESMF_FieldGet(packed_data(mapindex)%field_src, farrayptr=dataptr2d_packed, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Obtain unity normalization factor and multiply - ! interpolated field by reciprocal of normalization factor - if (trim(packed_data(mapindex)%mapnorm) == 'one') then - call ESMF_FieldGet(field_normOne(mapindex), farrayPtr=data_norm, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_FieldGet(packed_data(mapindex)%field_dst, farrayPtr=data_dst, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - do n = 1,size(data_dst,dim=2) - if (data_norm(n) == 0.0_r8) then - data_dst(:,n) = 0.0_r8 + ! Now do the copy + do nf = 1,fieldcount + ! Get the indices into the packed data structure + np = packed_data(mapindex)%fldindex(nf) + if (np > 0) then + call ESMF_FieldGet(fieldlist_src(nf), ungriddedUBound=ungriddedUBound, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (ungriddedUBound(1) > 0) then + call ESMF_FieldGet(fieldlist_src(nf), farrayptr=dataptr2d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + do nu = 1,ungriddedUBound(1) + dataptr2d_packed(np+nu-1,:) = dataptr2d(nu,:) + end do else - data_dst(:,n) = data_dst(:,n)/data_norm(n) + call ESMF_FieldGet(fieldlist_src(nf), farrayptr=dataptr1d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + dataptr2d_packed(np,:) = dataptr1d(:) end if - end do - end if + end if + end do + call t_stopf('MED:'//trim(subname)//' copy from src') - end if - call t_stopf('MED:'//trim(subname)//' map') + ! ----------------------------------- + ! Do the mapping + ! ----------------------------------- - ! ----------------------------------- - ! Copy the destination packed field bundle into the destination unpacked field bundle - ! ----------------------------------- + call t_startf('MED:'//trim(subname)//' map') - call t_startf('MED:'//trim(subname)//' copy to dest') + if (mapindex == mapfcopy) then - ! First get the pointer for the packed destination data - call ESMF_FieldGet(packed_data(mapindex)%field_dst, farrayptr=dataptr2d_packed, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return + ! Mapping is redistribution + call ESMF_FieldRedist(& + packed_data(mapindex)%field_src, & + packed_data(mapindex)%field_dst, & + routehandles(mapindex), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Now do the copy back to FBDst - do nf = 1,fieldcount - ! Get the indices into the packed data structure - np = packed_data(mapindex)%fldindex(nf) - if (np > 0) then - call ESMF_FieldGet(fieldlist_dst(nf), ungriddedUBound=ungriddedUBound, rc=rc) + else if ( trim(packed_data(mapindex)%mapnorm) /= 'unset' .and. & + trim(packed_data(mapindex)%mapnorm) /= 'one' .and. & + trim(packed_data(mapindex)%mapnorm) /= 'none') then + + ! Normalized mapping - assume that each packed field has only one normalization type + call ESMF_FieldBundleGet(FBFracSrc, packed_data(mapindex)%mapnorm, field=field_fracsrc, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - if (ungriddedUBound(1) > 0) then - call ESMF_FieldGet(fieldlist_dst(nf), farrayptr=dataptr2d, rc=rc) + call med_map_field_normalized(& + field_src=packed_data(mapindex)%field_src, & + field_dst=packed_data(mapindex)%field_dst, & + routehandles=routehandles, & + maptype=mapindex, & + field_normsrc=field_fracsrc, & + field_normdst=packed_data(mapindex)%field_fracdst, rc=rc) + + else if ( trim(packed_data(mapindex)%mapnorm) == 'one' .or. trim(packed_data(mapindex)%mapnorm) == 'none') then + + ! Mapping with no normalization that is not redistribution + call med_map_field (& + field_src=packed_data(mapindex)%field_src, & + field_dst=packed_data(mapindex)%field_dst, & + routehandles=routehandles, & + maptype=mapindex, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! Obtain unity normalization factor and multiply + ! interpolated field by reciprocal of normalization factor + if (trim(packed_data(mapindex)%mapnorm) == 'one') then + call ESMF_FieldGet(field_normOne(mapindex), farrayPtr=data_norm, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - do nu = 1,ungriddedUBound(1) - dataptr2d(nu,:) = dataptr2d_packed(np+nu-1,:) - end do - else - call ESMF_FieldGet(fieldlist_dst(nf), farrayptr=dataptr1d, rc=rc) + call ESMF_FieldGet(packed_data(mapindex)%field_dst, farrayPtr=data_dst, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - dataptr1d(:) = dataptr2d_packed(np,:) + do n = 1,size(data_dst,dim=2) + if (data_norm(n) == 0.0_r8) then + data_dst(:,n) = 0.0_r8 + else + data_dst(:,n) = data_dst(:,n)/data_norm(n) + end if + end do end if + end if - end do - call t_stopf('MED:'//trim(subname)//' copy to dest') + call t_stopf('MED:'//trim(subname)//' map') + + ! ----------------------------------- + ! Copy the destination packed field bundle into the destination unpacked field bundle + ! ----------------------------------- + + call t_startf('MED:'//trim(subname)//' copy to dest') + + ! First get the pointer for the packed destination data + call ESMF_FieldGet(packed_data(mapindex)%field_dst, farrayptr=dataptr2d_packed, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + ! Now do the copy back to FBDst + do nf = 1,fieldcount + ! Get the indices into the packed data structure + np = packed_data(mapindex)%fldindex(nf) + if (np > 0) then + call ESMF_FieldGet(fieldlist_dst(nf), ungriddedUBound=ungriddedUBound, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (ungriddedUBound(1) > 0) then + call ESMF_FieldGet(fieldlist_dst(nf), farrayptr=dataptr2d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + do nu = 1,ungriddedUBound(1) + dataptr2d(nu,:) = dataptr2d_packed(np+nu-1,:) + end do + else + call ESMF_FieldGet(fieldlist_dst(nf), farrayptr=dataptr1d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + dataptr1d(:) = dataptr2d_packed(np,:) + end if + end if + end do + call t_stopf('MED:'//trim(subname)//' copy to dest') + + end if end if end do ! end of loop over mapindex @@ -1154,24 +1155,26 @@ subroutine med_map_field(field_src, field_dst, routehandles, maptype, fldname, r end subroutine med_map_field !================================================================================ - subroutine med_map_uv_cart3d(usrc, vsrc, udst, vdst, routehandles, mapindex, rc) + subroutine med_map_uv_cart3d(FBsrc, FBdst, routehandles, mapindex, rc) use ESMF , only : ESMF_Mesh, ESMF_MeshGet, ESMF_MESHLOC_ELEMENT, ESMF_TYPEKIND_R8 - use ESMF , only : ESMF_Field, ESMF_FieldGet - use ESMF , only : ESMF_FieldCreate, ESMF_FieldDestroy, ESMF_FieldRegrid - use ESMF , only : ESMF_RouteHandle, ESMF_TERMORDER_SRCSEQ, ESMF_REGION_TOTAL + use ESMF , only : ESMF_Field, ESMF_FieldCreate, ESMF_FieldGet + use ESMF , only : ESMF_FieldBundle, ESMF_FieldBundleGet + use ESMF , only : ESMF_RouteHandle use shr_const_mod , only : shr_const_pi ! input/output variables - type(ESMF_Field) , intent(in) :: usrc - type(ESMF_Field) , intent(in) :: vsrc - type(ESMF_Field) , intent(inout) :: udst - type(ESMF_Field) , intent(inout) :: vdst + type(ESMF_FieldBundle) , intent(in) :: FBsrc + type(ESMF_FieldBundle) , intent(inout) :: FBdst type(ESMF_RouteHandle) , intent(inout) :: routehandles(:) integer , intent(in) :: mapindex integer , intent(out) :: rc ! local variables + type(ESMF_Field) :: usrc + type(ESMF_Field) :: vsrc + type(ESMF_Field) :: udst + type(ESMF_Field) :: vdst integer :: n real(r8) :: lon,lat real(r8) :: coslon,coslat @@ -1196,7 +1199,17 @@ subroutine med_map_uv_cart3d(usrc, vsrc, udst, vdst, routehandles, mapindex, rc) rc = ESMF_SUCCESS - ! Get pointer to input u and v data source field data + ! Get fields for atm u,v velocities + call ESMF_FieldBundleGet(FBSrc, fieldName='Sa_u', field=usrc, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldBundleGet(FBDst, fieldName='Sa_u', field=udst, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldBundleGet(FBSrc, fieldName='Sa_v', field=vsrc, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldBundleGet(FBDst, fieldName='Sa_v', field=vdst, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! GET pointer to input u and v data source field data call ESMF_FieldGet(usrc, farrayPtr=data_u_src, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return call ESMF_FieldGet(vsrc, farrayPtr=data_v_src, rc=rc) From 1c49a4b7900c72fe532c4b3106c08ac87bfb923d Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 2 Feb 2021 09:13:21 -0700 Subject: [PATCH 04/42] updated testlist_drv.xml --- cime_config/testdefs/testlist_drv.xml | 12 +- cime_config/testdefs/testlist_prealpha.xml | 529 --------------------- 2 files changed, 2 insertions(+), 539 deletions(-) delete mode 100644 cime_config/testdefs/testlist_prealpha.xml diff --git a/cime_config/testdefs/testlist_drv.xml b/cime_config/testdefs/testlist_drv.xml index d8c34b055..fcd1cd61c 100644 --- a/cime_config/testdefs/testlist_drv.xml +++ b/cime_config/testdefs/testlist_drv.xml @@ -145,15 +145,7 @@ - - - - - - - - - + @@ -163,7 +155,7 @@ - + diff --git a/cime_config/testdefs/testlist_prealpha.xml b/cime_config/testdefs/testlist_prealpha.xml deleted file mode 100644 index 9af907ac0..000000000 --- a/cime_config/testdefs/testlist_prealpha.xml +++ /dev/null @@ -1,529 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 4960f5fd7cea84cd6e79ca33816aacdc0fdfaa93 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 2 Feb 2021 09:44:54 -0700 Subject: [PATCH 05/42] removed hard coding of cnumber --- mediator/med_io_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 60746534a..0da4c95ff 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -457,7 +457,7 @@ subroutine med_io_write_FB(filename, iam, FB, whead, wdata, nx, ny, nt, & real(r8), pointer :: fldptr1(:) real(r8), pointer :: fldptr2(:,:) real(r8), allocatable :: ownedElemCoords(:), ownedElemCoords_x(:), ownedElemCoords_y(:) - character(16) :: cnumber + character(CS) :: cnumber character(CL) :: tmpstr type(ESMF_Field) :: lfield integer :: rank From 1a31e0dc9dfa775aeebe3c52811cb0fe723a6681 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 2 Feb 2021 09:59:26 -0700 Subject: [PATCH 06/42] removed DEBUG comments --- mediator/med_map_mod.F90 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 67636df48..24352bdca 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -357,10 +357,6 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & - !DEBUG - ! TODO: CREATE flddst_debug to do this - ! dststatusfield=flddst_debug, & - !DEBUG rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return else if (mapindex == mappatch .or. mapindex == mappatch_uv3d) then From e18514b4fb4f76f1982625fe2e5932c333e2383e Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Sun, 14 Feb 2021 00:54:22 -0600 Subject: [PATCH 07/42] update internal PIO initialization --- mediator/med.F90 | 2 +- mediator/med_io_mod.F90 | 309 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 290 insertions(+), 21 deletions(-) diff --git a/mediator/med.F90 b/mediator/med.F90 index 9b2bbcbef..493fac3ca 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -2389,7 +2389,7 @@ subroutine DataInitialize(gcomp, rc) !--------------------------------------- ! Initialize mediator IO !--------------------------------------- - call med_io_init() + call med_io_init(gcomp) !--------------------------------------- ! Initialize mediator water/heat budget diags diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 0a4fa7753..a9898f253 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -119,41 +119,310 @@ logical function med_io_file_exists(vm, iam, filename) end function med_io_file_exists !=============================================================================== - subroutine med_io_init() + subroutine med_io_init(gcomp) !--------------- ! initialize pio !--------------- +#ifdef CESMCOUPLED use shr_pio_mod , only : shr_pio_getiosys, shr_pio_getiotype, shr_pio_getioformat -#ifdef INTERNAL_PIO_INIT - ! if CMEPS is the only component using PIO, then it needs to initialize PIO - use shr_pio_mod , only : shr_pio_init1, shr_pio_init2 +#else + use pio , only : pio_init, pio_set_rearr_opts + use pio , only : PIO_64BIT_OFFSET, PIO_64BIT_DATA + use pio , only : PIO_IOTYPE_NETCDF, PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P + use pio , only : PIO_REARR_COMM_P2P, PIO_REARR_COMM_COLL + use pio , only : PIO_REARR_COMM_FC_2D_ENABLE, PIO_REARR_COMM_FC_2D_DISABLE + use pio , only : PIO_REARR_COMM_FC_1D_COMP2IO, PIO_REARR_COMM_FC_1D_IO2COMP + use ESMF , only : ESMF_GridComp, ESMF_UtilStringUpperCase + use NUOPC, only : NUOPC_CompAttributeGet +#endif + + ! input/output arguments + type(ESMF_GridComp), intent(in) :: gcomp - type(ESMF_VM) :: vm - integer :: comms(1), comps(1) - logical :: comp_iamin(1) - integer :: comp_comm_iam(1) - character(len=32) :: compLabels(1) - integer :: rc +#ifndef CESMCOUPLED + ! local variables + type(ESMF_VM) :: vm + integer :: rc + integer :: ret + integer :: comm + integer :: localPet, petCount + integer :: pio_numiotasks + integer :: pio_stride + integer :: pio_rearranger + integer :: pio_root + integer :: pio_rearr_comm_type + integer :: pio_rearr_comm_fcd + logical :: pio_rearr_comm_enable_hs_comp2io + logical :: pio_rearr_comm_enable_isend_comp2io + integer :: pio_rearr_comm_max_pend_req_comp2io + logical :: pio_rearr_comm_enable_hs_io2comp + logical :: pio_rearr_comm_enable_isend_io2comp + integer :: pio_rearr_comm_max_pend_req_io2comp + logical :: isPresent, isSet + character(len=CS) :: cvalue + character(*), parameter :: subname = '(med_io_init)' + !------------------------------------------------------------------------------- +#endif +#ifdef CESMCOUPLED + io_subsystem => shr_pio_getiosys(med_id) + pio_iotype = shr_pio_getiotype(med_id) + pio_ioformat = shr_pio_getioformat(med_id) +#else + ! query VM call ESMF_VMGetCurrent(vm=vm, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_VMGet(vm, mpiCommunicator=comms(1), localPet=comp_comm_iam(1), rc=rc) + call ESMF_VMGet(vm, mpiCommunicator=comm, localPet=localPet, petCount=petCount, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - comps(1) = med_id - compLabels(1) = "MED" - comp_iamin(1) = .true. + ! query component specific PIO attributes + ! pio_netcdf_format + call NUOPC_CompAttributeGet(gcomp, name='pio_netcdf_format', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + cvalue = ESMF_UtilStringUpperCase(cvalue) + if (trim(cvalue) .eq. 'CLASSIC') then + pio_ioformat = 0 + else if (trim(cvalue) .eq. '64BIT_OFFSET') then + pio_ioformat = PIO_64BIT_OFFSET + else if (trim(cvalue) .eq. '64BIT_DATA') then + pio_ioformat = PIO_64BIT_DATA + else + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_ioformat (CLASSIC|64BIT_OFFSET|64BIT_DATA)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + cvalue = '64BIT_OFFSET' + pio_ioformat = PIO_64BIT_OFFSET + end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_netcdf_format = ', trim(cvalue), pio_ioformat + + ! pio_typename + call NUOPC_CompAttributeGet(gcomp, name='pio_typename', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + cvalue = ESMF_UtilStringUpperCase(cvalue) + if (trim(cvalue) .eq. 'NETCDF') then + pio_iotype = PIO_IOTYPE_NETCDF + else if (trim(cvalue) .eq. 'PNETCDF') then + pio_iotype = PIO_IOTYPE_PNETCDF + else if (trim(cvalue) .eq. 'NETCDF4C') then + pio_iotype = PIO_IOTYPE_NETCDF4C + else if (trim(cvalue) .eq. 'NETCDF4P') then + pio_iotype = PIO_IOTYPE_NETCDF4P + else + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_typename (NETCDF|PNETCDF|NETCDF4C|NETCDF4P)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + cvalue = 'NETCDF' + pio_iotype = PIO_IOTYPE_NETCDF + end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_typename = ', trim(cvalue), pio_iotype + + ! pio_root + call NUOPC_CompAttributeGet(gcomp, name='pio_root', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) pio_root + if (pio_root < 0) then + pio_root = 1 + endif + pio_root = min(pio_root, petCount-1) + end if - call shr_pio_init1(1, "pio_in", comms(1)) - call shr_pio_init2(comps, compLabels, comp_iamin, comms, comp_comm_iam) -#endif + ! pio_stride + call NUOPC_CompAttributeGet(gcomp, name='pio_stride', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) pio_stride + end if - io_subsystem => shr_pio_getiosys(med_id) - pio_iotype = shr_pio_getiotype(med_id) - pio_ioformat = shr_pio_getioformat(med_id) + ! pio_numiotasks + call NUOPC_CompAttributeGet(gcomp, name='pio_numiotasks', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) pio_numiotasks + end if + + ! check for parallel IO, it requires at least two io pes + if (petCount > 1 .and. pio_numiotasks == 1 .and. & + (pio_iotype .eq. PIO_IOTYPE_PNETCDF .or. pio_iotype .eq. PIO_IOTYPE_NETCDF4P)) then + pio_numiotasks = 2 + pio_stride = min(pio_stride, petCount/2) + endif + + ! check/set/correct io pio parameters + if (pio_stride > 0 .and. pio_numiotasks < 0) then + pio_numiotasks = max(1, petCount/pio_stride) + else if(pio_numiotasks > 0 .and. pio_stride < 0) then + pio_stride = max(1, petCount/pio_numiotasks) + else if(pio_numiotasks < 0 .and. pio_stride < 0) then + pio_stride = max(1,petCount/4) + pio_numiotasks = max(1,petCount/pio_stride) + end if + if(pio_stride == 1) then + pio_root = 0 + endif + + if (pio_root + (pio_stride)*(pio_numiotasks-1) >= petCount .or. & + pio_stride <= 0 .or. pio_numiotasks <= 0 .or. pio_root < 0 .or. pio_root > petCount-1) then + if (petCount < 100) then + pio_stride = max(1, petCount/4) + else if(petCount < 1000) then + pio_stride = max(1, petCount/8) + else + pio_stride = max(1, petCount/16) + end if + if(pio_stride > 1) then + pio_numiotasks = petCount/pio_stride + pio_root = min(1, petCount-1) + else + pio_numiotasks = petCount + pio_root = 0 + end if + if (localPet == 0) then + write(logunit,*) 'pio_stride, iotasks or root out of bounds - resetting to defaults: ', & + pio_stride, pio_numiotasks, pio_root + end if + end if + + ! pio_rearranger + call NUOPC_CompAttributeGet(gcomp, name='pio_rearranger', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearranger + end if + + ! print out PIO parameters + if (localPet == 0) then + write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks + write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride + write(logunit,*) trim(subname), ' : pio_rearranger = ', pio_rearranger + write(logunit,*) trim(subname), ' : pio_root = ', pio_root + end if + + ! init PIO + allocate(io_subsystem) + if (localPet == 0) write(logunit,*) trim(subname),' calling pio init' + call pio_init(localPet, comm, pio_numiotasks, 0, pio_stride, pio_rearranger, io_subsystem, base=pio_root) + + ! query shared PIO rearranger attributes + ! pio_rearr_comm_type + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_type', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + cvalue = ESMF_UtilStringUpperCase(cvalue) + if (trim(cvalue) .eq. 'P2P') then + pio_rearr_comm_type = PIO_REARR_COMM_P2P + else if (trim(cvalue) .eq. 'COLL') then + pio_rearr_comm_type = PIO_REARR_COMM_COLL + else + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearr_comm_type (P2P|COLL)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + pio_rearr_comm_type = PIO_REARR_COMM_P2P + end if + + ! pio_rearr_comm_fcd + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_fcd', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + cvalue = ESMF_UtilStringUpperCase(cvalue) + if (trim(cvalue) .eq. '2DENABLE') then + pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_ENABLE + else if (trim(cvalue) .eq. 'IO2COMP') then + pio_rearr_comm_fcd = PIO_REARR_COMM_FC_1D_IO2COMP + else if (trim(cvalue) .eq. 'COMP2IO') then + pio_rearr_comm_fcd = PIO_REARR_COMM_FC_1D_COMP2IO + else if (trim(cvalue) .eq. '2DDISABLE') then + pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_DISABLE + else + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearr_comm_fcd (2DENABLE|IO2COMP|COMP2IO|2DDISABLE)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_ENABLE + end if + + ! pio_rearr_comm_enable_hs_comp2io + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_hs_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_enable_hs_comp2io + else + pio_rearr_comm_enable_hs_comp2io = .true. + end if + + ! pio_rearr_comm_enable_isend_comp2io + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_isend_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_enable_isend_comp2io + else + pio_rearr_comm_enable_isend_comp2io = .false. + end if + + ! pio_rearr_comm_max_pend_req_comp2io + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_max_pend_req_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_max_pend_req_comp2io + else + pio_rearr_comm_max_pend_req_comp2io = 0 + end if + + ! pio_rearr_comm_enable_hs_io2comp + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_hs_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_enable_hs_io2comp + else + pio_rearr_comm_enable_hs_io2comp = .false. + end if + + ! pio_rearr_comm_enable_isend_io2comp + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_isend_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_enable_isend_io2comp + else + pio_rearr_comm_enable_isend_io2comp = .true. + end if + + ! pio_rearr_comm_max_pend_req_io2comp + call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_max_pend_req_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_rearr_comm_max_pend_req_io2comp + else + pio_rearr_comm_max_pend_req_io2comp = 0 + end if + + ! set PIO rearranger options + if (localPet == 0) write(logunit,*) trim(subname),' calling pio_set_rearr_opts' + ret = pio_set_rearr_opts(io_subsystem, pio_rearr_comm_type, pio_rearr_comm_fcd, & + pio_rearr_comm_enable_hs_comp2io, pio_rearr_comm_enable_isend_comp2io, & + pio_rearr_comm_max_pend_req_comp2io, & + pio_rearr_comm_enable_hs_io2comp, pio_rearr_comm_enable_isend_io2comp, & + pio_rearr_comm_max_pend_req_io2comp) +#endif end subroutine med_io_init From 2a1977acb1e033591984bb773166eb5b1a992de3 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 16 Feb 2021 14:50:45 -0600 Subject: [PATCH 08/42] more work related to component level PIO initialization --- mediator/med_io_mod.F90 | 43 +- util/shr_pio_mod.F90 | 910 ---------------------------------------- 2 files changed, 38 insertions(+), 915 deletions(-) delete mode 100644 util/shr_pio_mod.F90 diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index a9898f253..8e69fec60 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -131,6 +131,7 @@ subroutine med_io_init(gcomp) use pio , only : pio_init, pio_set_rearr_opts use pio , only : PIO_64BIT_OFFSET, PIO_64BIT_DATA use pio , only : PIO_IOTYPE_NETCDF, PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P + use pio , only : PIO_REARR_BOX, PIO_REARR_SUBSET use pio , only : PIO_REARR_COMM_P2P, PIO_REARR_COMM_COLL use pio , only : PIO_REARR_COMM_FC_2D_ENABLE, PIO_REARR_COMM_FC_2D_DISABLE use pio , only : PIO_REARR_COMM_FC_1D_COMP2IO, PIO_REARR_COMM_FC_1D_IO2COMP @@ -234,6 +235,8 @@ subroutine med_io_init(gcomp) pio_root = 1 endif pio_root = min(pio_root, petCount-1) + else + pio_root = -99 end if ! pio_stride @@ -241,6 +244,8 @@ subroutine med_io_init(gcomp) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (isPresent .and. isSet) then read(cvalue,*) pio_stride + else + pio_stride = -99 end if ! pio_numiotasks @@ -248,6 +253,8 @@ subroutine med_io_init(gcomp) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (isPresent .and. isSet) then read(cvalue,*) pio_numiotasks + else + pio_numiotasks = -99 end if ! check for parallel IO, it requires at least two io pes @@ -296,14 +303,26 @@ subroutine med_io_init(gcomp) call NUOPC_CompAttributeGet(gcomp, name='pio_rearranger', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (isPresent .and. isSet) then - read(cvalue,*) pio_rearranger + cvalue = ESMF_UtilStringUpperCase(cvalue) + if (trim(cvalue) .eq. 'BOX') then + pio_rearranger = PIO_REARR_BOX + else if (trim(cvalue) .eq. 'SUBSET') then + pio_rearranger = PIO_REARR_SUBSET + else + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearranger (BOX|SUBSET)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + cvalue = 'SUBSET' + pio_rearranger = PIO_REARR_SUBSET end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearranger = ', trim(cvalue), pio_rearranger - ! print out PIO parameters + ! print out PIO init parameters if (localPet == 0) then write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride - write(logunit,*) trim(subname), ' : pio_rearranger = ', pio_rearranger write(logunit,*) trim(subname), ' : pio_root = ', pio_root end if @@ -329,8 +348,10 @@ subroutine med_io_init(gcomp) return end if else + cvalue = 'P2P' pio_rearr_comm_type = PIO_REARR_COMM_P2P end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_type = ', trim(cvalue), pio_rearr_comm_type ! pio_rearr_comm_fcd call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_fcd', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) @@ -352,8 +373,10 @@ subroutine med_io_init(gcomp) return end if else + cvalue = '2DENABLE' pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_ENABLE - end if + end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_fcd = ', trim(cvalue), pio_rearr_comm_fcd ! pio_rearr_comm_enable_hs_comp2io call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_hs_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) @@ -412,7 +435,17 @@ subroutine med_io_init(gcomp) if (isPresent .and. isSet) then read(cvalue,*) pio_rearr_comm_max_pend_req_io2comp else - pio_rearr_comm_max_pend_req_io2comp = 0 + pio_rearr_comm_max_pend_req_io2comp = 64 + end if + + ! print out PIO rearranger parameters + if (localPet == 0) then + write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_hs_comp2io = ', pio_rearr_comm_enable_hs_comp2io + write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_isend_comp2io = ', pio_rearr_comm_enable_isend_comp2io + write(logunit,*) trim(subname), ' : pio_rearr_comm_max_pend_req_comp2io = ', pio_rearr_comm_max_pend_req_comp2io + write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_hs_io2comp = ', pio_rearr_comm_enable_hs_io2comp + write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_isend_io2comp = ', pio_rearr_comm_enable_isend_io2comp + write(logunit,*) trim(subname), ' : pio_rearr_comm_max_pend_req_io2comp = ', pio_rearr_comm_max_pend_req_io2comp end if ! set PIO rearranger options diff --git a/util/shr_pio_mod.F90 b/util/shr_pio_mod.F90 deleted file mode 100644 index a88fd2d02..000000000 --- a/util/shr_pio_mod.F90 +++ /dev/null @@ -1,910 +0,0 @@ -module shr_pio_mod - - use pio - use ESMF , only : ESMF_UtilStringLowerCase, ESMF_UtilStringUpperCase - use shr_kind_mod , only : shr_kind_CS, shr_kind_cl, shr_kind_in - use shr_log_mod , only : shr_log_unit - use shr_mpi_mod , only : shr_mpi_bcast, shr_mpi_chkerr - use shr_sys_mod , only : shr_sys_abort -#ifndef NO_MPIMOD - use mpi , only : mpi_comm_null, mpi_comm_world -#endif - implicit none -#ifdef NO_MPIMOD -#include -#endif - - private - public :: shr_pio_init1 - public :: shr_pio_init2 - public :: shr_pio_getiosys - public :: shr_pio_getiotype - public :: shr_pio_getioroot - public :: shr_pio_finalize - public :: shr_pio_getioformat - public :: shr_pio_getrearranger - - interface shr_pio_getiotype - module procedure shr_pio_getiotype_fromid, shr_pio_getiotype_fromname - end interface - interface shr_pio_getioformat - module procedure shr_pio_getioformat_fromid, shr_pio_getioformat_fromname - end interface - interface shr_pio_getiosys - module procedure shr_pio_getiosys_fromid, shr_pio_getiosys_fromname - end interface - interface shr_pio_getioroot - module procedure shr_pio_getioroot_fromid, shr_pio_getioroot_fromname - end interface - interface shr_pio_getindex - module procedure shr_pio_getindex_fromid, shr_pio_getindex_fromname - end interface - interface shr_pio_getrearranger - module procedure shr_pio_getrearranger_fromid, shr_pio_getrearranger_fromname - end interface - - type pio_comp_t - integer :: compid - integer :: pio_root - integer :: pio_stride - integer :: pio_numiotasks - integer :: pio_iotype - integer :: pio_rearranger - integer :: pio_netcdf_ioformat - end type pio_comp_t - - character(len=16), allocatable :: io_compname(:) - type(pio_comp_t), allocatable :: pio_comp_settings(:) - type (iosystem_desc_t), allocatable, target :: iosystems(:) - integer :: io_comm - logical :: pio_async_interface - integer, allocatable :: io_compid(:) - integer :: pio_debug_level=0, pio_blocksize=0 - integer(kind=pio_offset_kind) :: pio_buffer_size_limit=-1 - integer :: pio_rearr_opt_comm_type, pio_rearr_opt_fcd - logical :: pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend - integer :: pio_rearr_opt_c2i_max_pend_req - logical :: pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend - integer :: pio_rearr_opt_i2c_max_pend_req - integer :: total_comps=0 - -#define DEBUGI 1 - -#ifdef DEBUGI - integer :: drank -#endif - - -contains -!> -!! @public -!! @brief should be the first routine called after mpi_init. -!! It reads the pio default settings from file drv_in, namelist pio_default_inparm -!! and, if pio_async_interface is true, splits the IO tasks away from the -!! Compute tasks. It then returns the new compute comm in -!! Global_Comm and sets module variable io_comm. -!! -!< - subroutine shr_pio_init1(ncomps, nlfilename, Global_Comm) - integer, intent(in) :: ncomps - character(len=*) :: nlfilename - integer, intent(inout) :: Global_Comm - - - integer :: i, pio_root, pio_stride, pio_numiotasks, pio_iotype, pio_rearranger, pio_netcdf_ioformat - integer :: mpigrp_world, mpigrp, ierr, mpicom - character(*),parameter :: subName = '(shr_pio_init1) ' - integer :: pelist(3,1) - - call shr_pio_read_default_namelist(nlfilename, Global_Comm, pio_stride, pio_root, pio_numiotasks, & - pio_iotype, pio_async_interface, pio_rearranger) - - pio_netcdf_ioformat = PIO_64BIT_OFFSET - call MPI_comm_rank(Global_Comm, drank, ierr) - - io_comm = MPI_COMM_NULL - if(.not.allocated(pio_comp_settings))allocate(pio_comp_settings(ncomps)) - do i=1,ncomps - pio_comp_settings(i)%pio_root = pio_root - pio_comp_settings(i)%pio_stride = pio_stride - pio_comp_settings(i)%pio_numiotasks = pio_numiotasks - pio_comp_settings(i)%pio_iotype = pio_iotype - pio_comp_settings(i)%pio_rearranger = pio_rearranger - pio_comp_settings(i)%pio_netcdf_ioformat = pio_netcdf_ioformat - end do - if(pio_async_interface) then -#ifdef NO_MPI2 - call shr_sys_abort(subname//':: async IO requires an MPI2 compliant MPI library') -#else - - pelist(1,1) = pio_root - pelist(2,1) = pio_root + (pio_numiotasks-1)*pio_stride - pelist(3,1) = pio_stride - - call mpi_comm_group(GLOBAL_COMM, mpigrp_world, ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_group mpigrp_world') - call mpi_group_range_incl(mpigrp_world, 1, pelist, mpigrp,ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_group_range_incl mpigrp') - call mpi_comm_create(GLOBAL_COMM, mpigrp, io_comm, ierr) - - call mpi_group_range_excl(mpigrp_world, 1, pelist, mpigrp,ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_group_range_incl mpigrp') - call mpi_comm_create(GLOBAL_COMM, mpigrp, mpicom, ierr) - Global_COMM=mpicom - - print *,__FILE__,__LINE__,subname, ' complete' -#endif - end if - total_comps = ncomps - end subroutine shr_pio_init1 -!> -!! @public -!! @brief if pio_async_interface is true, tasks in io_comm do not return from this subroutine. -!! -!! if pio_async_interface is false each component namelist pio_inparm is read from compname_modelio.nml -!! Then a subset of each components compute tasks are Identified as IO tasks using the root, stride and count -!! variables to select the tasks. -!! -!< - - - subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_iam) - - integer , intent(in) :: comp_id(:) - logical , intent(in) :: comp_iamin(:) - character(len=*) , intent(in) :: comp_name(:) - integer , intent(in) :: comp_comm(:), comp_comm_iam(:) - - integer :: i - character(len=shr_kind_cl) :: nlfilename, cname - integer :: ret - character(*), parameter :: subName = '(shr_pio_init2) ' - - if(pio_debug_level>0) then - if(comp_comm_iam(1)==0) then - write(shr_log_unit,*) 'Setting pio_debuglevel : ',pio_debug_level - end if - call pio_setdebuglevel(pio_debug_level) - endif - ! 0 is a valid value of pio_buffer_size_limit - if(pio_buffer_size_limit>=0) then - if(comp_comm_iam(1)==0) then - write(shr_log_unit,*) 'Setting pio_buffer_size_limit : ',pio_buffer_size_limit - end if - call pio_set_buffer_size_limit(pio_buffer_size_limit) - endif - if(pio_blocksize>0) then - if(comp_comm_iam(1)==0) then - write(shr_log_unit,*) 'Setting pio_blocksize : ',pio_blocksize - end if - call pio_set_blocksize(pio_blocksize) - endif - ! Correct the total_comps value which may be lower in nuopc - total_comps = size(comp_iamin) - - - allocate(io_compid(total_comps), io_compname(total_comps)) - - io_compid = comp_id - io_compname = comp_name - allocate(iosystems(total_comps)) - - if(pio_async_interface) then - call shr_sys_abort('pio_async_interface is not currently supported') -! call pio_init(total_comps,mpi_comm_world, comp_comm, io_comm, iosystems) -! do i=1,total_comps -! ret = pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,& -! pio_rearr_opt_fcd,& -! pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,& -! pio_rearr_opt_c2i_max_pend_req,& -! pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,& -! pio_rearr_opt_i2c_max_pend_req) -! if(ret /= PIO_NOERR) then -! write(shr_log_unit,*) "ERROR: Setting rearranger options failed" -! end if -! end do -! i=1 - else - do i=1,total_comps - if(comp_iamin(i)) then - cname = comp_name(i) - if(len_trim(cname) <= 3) then - nlfilename=trim(ESMF_UtilStringLowerCase(cname))//'_modelio.nml' - else - nlfilename=trim(ESMF_UtilStringLowerCase(cname(1:3)))//'_modelio.nml_'//cname(4:8) - endif - - call shr_pio_read_component_namelist(nlfilename , comp_comm(i), pio_comp_settings(i)%pio_stride, & - pio_comp_settings(i)%pio_root, pio_comp_settings(i)%pio_numiotasks, & - pio_comp_settings(i)%pio_iotype, pio_comp_settings(i)%pio_rearranger, & - pio_comp_settings(i)%pio_netcdf_ioformat) - call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, & - pio_comp_settings(i)%pio_stride, & - pio_comp_settings(i)%pio_rearranger, iosystems(i), & - base=pio_comp_settings(i)%pio_root) - ret = pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,& - pio_rearr_opt_fcd,& - pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,& - pio_rearr_opt_c2i_max_pend_req,& - pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,& - pio_rearr_opt_i2c_max_pend_req) - if(ret /= PIO_NOERR) then - write(shr_log_unit,*) "ERROR: Setting rearranger options failed" - end if - end if - end do - end if - do i=1,total_comps - if(comp_iamin(i) .and. (comp_comm_iam(i) == 0)) then - write(shr_log_unit,*) io_compname(i),' : pio_numiotasks = ',pio_comp_settings(i)%pio_numiotasks - write(shr_log_unit,*) io_compname(i),' : pio_stride = ',pio_comp_settings(i)%pio_stride - write(shr_log_unit,*) io_compname(i),' : pio_rearranger = ',pio_comp_settings(i)%pio_rearranger - write(shr_log_unit,*) io_compname(i),' : pio_root = ',pio_comp_settings(i)%pio_root - write(shr_log_unit,*) io_compname(i),' : pio_iotype = ',pio_comp_settings(i)%pio_iotype - end if - enddo - - end subroutine shr_pio_init2 - - - -!=============================================================================== - subroutine shr_pio_finalize( ) - integer :: ierr - integer :: i - do i=1,total_comps - call pio_finalize(iosystems(i), ierr) - end do - - end subroutine shr_pio_finalize - -!=============================================================================== - function shr_pio_getiotype_fromid(compid) result(io_type) - integer, intent(in) :: compid - integer :: io_type - - io_type = pio_comp_settings(shr_pio_getindex(compid))%pio_iotype - - end function shr_pio_getiotype_fromid - - - function shr_pio_getiotype_fromname(component) result(io_type) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - integer :: io_type - - io_type = pio_comp_settings(shr_pio_getindex(component))%pio_iotype - - end function shr_pio_getiotype_fromname - - function shr_pio_getrearranger_fromid(compid) result(io_type) - integer, intent(in) :: compid - integer :: io_type - - io_type = pio_comp_settings(shr_pio_getindex(compid))%pio_rearranger - - end function shr_pio_getrearranger_fromid - - - function shr_pio_getrearranger_fromname(component) result(io_type) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - integer :: io_type - - io_type = pio_comp_settings(shr_pio_getindex(component))%pio_rearranger - - end function shr_pio_getrearranger_fromname - - function shr_pio_getioformat_fromid(compid) result(io_format) - integer, intent(in) :: compid - integer :: io_format - - io_format = pio_comp_settings(shr_pio_getindex(compid))%pio_netcdf_ioformat - - end function shr_pio_getioformat_fromid - - - function shr_pio_getioformat_fromname(component) result(io_format) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - integer :: io_format - - io_format = pio_comp_settings(shr_pio_getindex(component))%pio_netcdf_ioformat - - end function shr_pio_getioformat_fromname - -!=============================================================================== - function shr_pio_getioroot_fromid(compid) result(io_root) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - integer, intent(in) :: compid - integer :: io_root - - io_root = pio_comp_settings(shr_pio_getindex(compid))%pio_root - - end function shr_pio_getioroot_fromid - - function shr_pio_getioroot_fromname(component) result(io_root) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - integer :: io_root - - io_root = pio_comp_settings(shr_pio_getindex(component))%pio_root - - - end function shr_pio_getioroot_fromname - - -!=============================================================================== - - !! Given a component name, return the index of that component. - !! This is the index into io_compid, io_compname, comp_pio_iotype, etc. - !! If the given component is not found, return -1 - - integer function shr_pio_getindex_fromid(compid) result(index) - implicit none - integer, intent(in) :: compid - integer :: i - - index = -1 - do i=1,total_comps - if(io_compid(i)==compid) then - index = i - exit - end if - end do - - if(index<0) then - call shr_sys_abort('shr_pio_getindex :: compid out of allowed range') - end if - end function shr_pio_getindex_fromid - - - integer function shr_pio_getindex_fromname(component) result(index) - - implicit none - - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - - character(len=len(component)) :: component_ucase - integer :: i - - ! convert component name to upper case in order to match case in io_compname - component_ucase = ESMF_UtilStringUpperCase(component) - - index = -1 ! flag for not found - do i=1,size(io_compname) - if (trim(component_ucase) == trim(io_compname(i))) then - index = i - exit - end if - end do - if(index<0) then - call shr_sys_abort(' shr_pio_getindex:: compid out of allowed range') - end if - end function shr_pio_getindex_fromname - - function shr_pio_getiosys_fromid(compid) result(iosystem) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - integer, intent(in) :: compid - type(iosystem_desc_t), pointer :: iosystem - - - iosystem => iosystems(shr_pio_getindex(compid)) - - end function shr_pio_getiosys_fromid - - function shr_pio_getiosys_fromname(component) result(iosystem) - ! 'component' must be equal to some element of io_compname(:) - ! (but it is case-insensitive) - character(len=*), intent(in) :: component - type(iosystem_desc_t), pointer :: iosystem - - iosystem => iosystems(shr_pio_getindex(component)) - - end function shr_pio_getiosys_fromname - -!=============================================================================== - - - - subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root, pio_numiotasks, & - pio_iotype, pio_async_interface, pio_rearranger) - - character(len=*), intent(in) :: nlfilename - integer, intent(in) :: Comm - logical, intent(out) :: pio_async_interface - integer, intent(out) :: pio_stride, pio_root, pio_numiotasks, pio_iotype, pio_rearranger - - character(len=shr_kind_cs) :: pio_typename - character(len=shr_kind_cs) :: pio_rearr_comm_type, pio_rearr_comm_fcd - integer :: pio_netcdf_ioformat - integer :: pio_rearr_comm_max_pend_req_comp2io - logical :: pio_rearr_comm_enable_hs_comp2io, pio_rearr_comm_enable_isend_comp2io - integer :: pio_rearr_comm_max_pend_req_io2comp - logical :: pio_rearr_comm_enable_hs_io2comp, pio_rearr_comm_enable_isend_io2comp - character(*),parameter :: subName = '(shr_pio_read_default_namelist) ' - - integer :: iam, ierr, npes, unitn - logical :: iamroot - namelist /pio_default_inparm/ & - pio_async_interface, pio_debug_level, pio_blocksize, & - pio_buffer_size_limit, & - pio_rearr_comm_type, pio_rearr_comm_fcd, & - pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, & - pio_rearr_comm_enable_isend_comp2io, & - pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, & - pio_rearr_comm_enable_isend_io2comp - - - call mpi_comm_rank(Comm, iam , ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_rank comm_world') - call mpi_comm_size(Comm, npes, ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_size comm_world') - - if(iam==0) then - iamroot=.true. - else - iamroot=.false. - end if - - !-------------------------------------------------------------------------- - ! read io nml parameters - !-------------------------------------------------------------------------- - pio_stride = -99 ! set based on pio_numiotasks value when initialized < 0 - pio_numiotasks = -99 ! set based on pio_stride value when initialized < 0 - pio_root = -99 - pio_typename = 'nothing' - pio_blocksize= -99 ! io blocking size set internally in pio when < 0 - pio_buffer_size_limit = -99 ! io task memory buffer maximum set internally in pio when < 0 - pio_debug_level = 0 ! no debug info by default - pio_async_interface = .false. ! pio tasks are a subset of component tasks - pio_rearranger = PIO_REARR_SUBSET - pio_netcdf_ioformat = PIO_64BIT_OFFSET - pio_rearr_comm_type = 'p2p' - pio_rearr_comm_fcd = '2denable' - pio_rearr_comm_max_pend_req_comp2io = 0 - pio_rearr_comm_enable_hs_comp2io = .true. - pio_rearr_comm_enable_isend_comp2io = .false. - pio_rearr_comm_max_pend_req_io2comp = 0 - pio_rearr_comm_enable_hs_io2comp = .true. - pio_rearr_comm_enable_isend_io2comp = .false. - - if(iamroot) then - open(newunit=unitn, file=trim(nlfilename), status='old' , iostat=ierr) - if(ierr/=0) then - write(shr_log_unit,*) 'File ',trim(nlfilename),' not found, setting default values.' - else - ierr = 1 - do while( ierr /= 0 ) - read(unitn,nml=pio_default_inparm,iostat=ierr) - if (ierr < 0) then - call shr_sys_abort( subname//':: namelist read returns an'// & - ' end of file or end of record condition '//trim(nlfilename) ) - end if - end do - close(unitn) - - call shr_pio_getiotypefromname(pio_typename, pio_iotype, pio_iotype_netcdf) - end if - end if - - call shr_pio_namelist_set(npes, Comm, pio_stride, pio_root, pio_numiotasks, pio_iotype, & - iamroot, pio_rearranger, pio_netcdf_ioformat) - - call shr_mpi_bcast(pio_debug_level, Comm) - call shr_mpi_bcast(pio_blocksize, Comm) - call shr_mpi_bcast(pio_buffer_size_limit, Comm) - call shr_mpi_bcast(pio_async_interface, Comm) - call shr_mpi_bcast(pio_rearranger, Comm) - - call shr_pio_rearr_opts_set(Comm, pio_rearr_comm_type, pio_rearr_comm_fcd, & - pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, & - pio_rearr_comm_enable_isend_comp2io, & - pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, & - pio_rearr_comm_enable_isend_io2comp, pio_numiotasks) - - end subroutine shr_pio_read_default_namelist - - subroutine shr_pio_read_component_namelist(nlfilename, Comm, pio_stride, pio_root, & - pio_numiotasks, pio_iotype, pio_rearranger, pio_netcdf_ioformat) - character(len=*), intent(in) :: nlfilename - integer, intent(in) :: Comm - - integer, intent(inout) :: pio_stride, pio_root, pio_numiotasks - integer, intent(inout) :: pio_iotype, pio_rearranger, pio_netcdf_ioformat - character(len=SHR_KIND_CS) :: pio_typename - character(len=SHR_KIND_CS) :: pio_netcdf_format - integer :: unitn - - integer :: iam, ierr, npes - logical :: iamroot - character(*),parameter :: subName = '(shr_pio_read_component_namelist) ' - integer :: pio_default_stride, pio_default_root, pio_default_numiotasks, pio_default_iotype - integer :: pio_default_rearranger, pio_default_netcdf_ioformat - - namelist /pio_inparm/ pio_stride, pio_root, pio_numiotasks, & - pio_typename, pio_rearranger, pio_netcdf_format - - - - call mpi_comm_rank(Comm, iam , ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_rank comm_world') - call mpi_comm_size(Comm, npes, ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_size comm_world') - - if(iam==0) then - iamroot=.true. - else - iamroot=.false. - end if - - pio_default_stride = pio_stride - pio_default_root = pio_root - pio_default_numiotasks = pio_numiotasks - pio_default_iotype = pio_iotype - pio_default_rearranger = pio_rearranger - pio_default_netcdf_ioformat = PIO_64BIT_OFFSET - - !-------------------------------------------------------------------------- - ! read io nml parameters - !-------------------------------------------------------------------------- - pio_stride = -99 ! set based on pio_numiotasks value when initialized < 0 - pio_numiotasks = -99 ! set based on pio_stride value when initialized < 0 - pio_root = -99 - pio_typename = 'nothing' - pio_rearranger = -99 - pio_netcdf_format = '64bit_offset' - - if(iamroot) then - open(newunit=unitn, file=trim(nlfilename), status='old' , iostat=ierr) - if( ierr /= 0) then - write(shr_log_unit,*) 'No ',trim(nlfilename),' found, using defaults for pio settings' - pio_stride = pio_default_stride - pio_root = pio_default_root - pio_numiotasks = pio_default_numiotasks - pio_iotype = pio_default_iotype - pio_rearranger = pio_default_rearranger - pio_netcdf_ioformat = pio_default_netcdf_ioformat - else - ierr = 1 - do while( ierr /= 0 ) - read(unitn,nml=pio_inparm,iostat=ierr) - if (ierr < 0) then - call shr_sys_abort( subname//':: namelist read returns an'// & - ' end of file or end of record condition' ) - end if - end do - close(unitn) - - call shr_pio_getiotypefromname(pio_typename, pio_iotype, pio_default_iotype) - call shr_pio_getioformatfromname(pio_netcdf_format, pio_netcdf_ioformat, pio_default_netcdf_ioformat) - end if - if(pio_stride== -99) then - if (pio_numiotasks > 0) then - pio_stride = npes/pio_numiotasks - else - pio_stride = pio_default_stride - endif - endif - if(pio_root == -99) pio_root = pio_default_root - if(pio_rearranger == -99) pio_rearranger = pio_default_rearranger - if(pio_numiotasks == -99) then - pio_numiotasks = npes/pio_stride - endif - endif - - - - call shr_pio_namelist_set(npes, Comm, pio_stride, pio_root, pio_numiotasks, pio_iotype, & - iamroot, pio_rearranger, pio_netcdf_ioformat) - - - end subroutine shr_pio_read_component_namelist - - subroutine shr_pio_getioformatfromname(pio_netcdf_format, pio_netcdf_ioformat, pio_default_netcdf_ioformat) - - character(len=*), intent(inout) :: pio_netcdf_format - integer, intent(out) :: pio_netcdf_ioformat - integer, intent(in) :: pio_default_netcdf_ioformat - - pio_netcdf_format = ESMF_UtilStringUpperCase(pio_netcdf_format) - if ( pio_netcdf_format .eq. 'CLASSIC' ) then - pio_netcdf_ioformat = 0 - elseif ( pio_netcdf_format .eq. '64BIT_OFFSET' ) then - pio_netcdf_ioformat = PIO_64BIT_OFFSET -#ifdef _PNETCDF - elseif ( pio_netcdf_format .eq. '64BIT_DATA' ) then - pio_netcdf_ioformat = PIO_64BIT_DATA -#endif - else - pio_netcdf_ioformat = pio_default_netcdf_ioformat - endif - - end subroutine shr_pio_getioformatfromname - - - subroutine shr_pio_getiotypefromname(typename, iotype, defaulttype) - - character(len=*), intent(inout) :: typename - integer, intent(out) :: iotype - integer, intent(in) :: defaulttype - - typename = ESMF_UtilStringUpperCase(typename) - if ( typename .eq. 'NETCDF' ) then - iotype = pio_iotype_netcdf - else if ( typename .eq. 'PNETCDF') then - iotype = pio_iotype_pnetcdf - else if ( typename .eq. 'NETCDF4P') then - iotype = pio_iotype_netcdf4p - else if ( typename .eq. 'NETCDF4C') then - iotype = pio_iotype_netcdf4c - else if ( typename .eq. 'NOTHING') then - iotype = defaulttype - else if ( typename .eq. 'DEFAULT') then - iotype = defaulttype - else - write(shr_log_unit,*) 'shr_pio_mod: WARNING Bad io_type argument - using iotype_netcdf' - iotype=pio_iotype_netcdf - end if - - end subroutine shr_pio_getiotypefromname - -!=============================================================================== - subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotasks, & - pio_iotype, iamroot, pio_rearranger, pio_netcdf_ioformat) - integer, intent(in) :: npes, mycomm - integer, intent(inout) :: pio_stride, pio_root, pio_numiotasks - integer, intent(inout) :: pio_iotype, pio_rearranger, pio_netcdf_ioformat - logical, intent(in) :: iamroot - character(*),parameter :: subName = '(shr_pio_namelist_set) ' - - call shr_mpi_bcast(pio_iotype , mycomm) - call shr_mpi_bcast(pio_stride , mycomm) - call shr_mpi_bcast(pio_root , mycomm) - call shr_mpi_bcast(pio_numiotasks, mycomm) - call shr_mpi_bcast(pio_rearranger, mycomm) - call shr_mpi_bcast(pio_netcdf_ioformat, mycomm) - - if (pio_root<0) then - pio_root = 1 - endif - pio_root = min(pio_root,npes-1) - -! If you are asking for parallel IO then you should use at least two io pes - if(npes > 1 .and. pio_numiotasks == 1 .and. & - (pio_iotype .eq. PIO_IOTYPE_PNETCDF .or. & - pio_iotype .eq. PIO_IOTYPE_NETCDF4P)) then - pio_numiotasks = 2 - pio_stride = min(pio_stride, npes/2) - endif - - !-------------------------------------------------------------------------- - ! check/set/correct io pio parameters - !-------------------------------------------------------------------------- - if (pio_stride>0.and.pio_numiotasks<0) then - pio_numiotasks = max(1,npes/pio_stride) - else if(pio_numiotasks>0 .and. pio_stride<0) then - pio_stride = max(1,npes/pio_numiotasks) - else if(pio_numiotasks<0 .and. pio_stride<0) then - pio_stride = max(1,npes/4) - pio_numiotasks = max(1,npes/pio_stride) - end if - if(pio_stride == 1) then - pio_root = 0 - endif - if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX) then - write(shr_log_unit,*) 'pio_rearranger value, ',pio_rearranger,& - ', not supported - using PIO_REARR_BOX' - pio_rearranger = PIO_REARR_BOX - - endif - - - if (pio_root + (pio_stride)*(pio_numiotasks-1) >= npes .or. & - pio_stride<=0 .or. pio_numiotasks<=0 .or. pio_root < 0 .or. & - pio_root > npes-1) then - if(npes<100) then - pio_stride = max(1,npes/4) - else if(npes<1000) then - pio_stride = max(1,npes/8) - else - pio_stride = max(1,npes/16) - end if - if(pio_stride>1) then - pio_numiotasks = npes/pio_stride - pio_root = min(1,npes-1) - else - pio_numiotasks = npes - pio_root = 0 - end if - if( iamroot) then - write(shr_log_unit,*) 'pio_stride, iotasks or root out of bounds - resetting to defaults: ',& - pio_stride,pio_numiotasks, pio_root - end if - end if - - end subroutine shr_pio_namelist_set - - ! This subroutine sets the global PIO rearranger options - ! The input args that represent the rearranger options are valid only - ! on the root proc of comm - ! The rearranger options are passed to PIO_Init() in shr_pio_init2() - subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd, & - pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, & - pio_rearr_comm_enable_isend_comp2io, & - pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, & - pio_rearr_comm_enable_isend_io2comp, & - pio_numiotasks) - integer(SHR_KIND_IN), intent(in) :: comm - character(len=shr_kind_cs), intent(in) :: pio_rearr_comm_type, pio_rearr_comm_fcd - integer, intent(in) :: pio_rearr_comm_max_pend_req_comp2io - logical, intent(in) :: pio_rearr_comm_enable_hs_comp2io - logical, intent(in) :: pio_rearr_comm_enable_isend_comp2io - integer, intent(in) :: pio_rearr_comm_max_pend_req_io2comp - logical, intent(in) :: pio_rearr_comm_enable_hs_io2comp - logical, intent(in) :: pio_rearr_comm_enable_isend_io2comp - integer, intent(in) :: pio_numiotasks - - character(*), parameter :: subname = '(shr_pio_rearr_opts_set) ' - integer, parameter :: NUM_REARR_COMM_OPTS = 8 - integer, parameter :: PIO_REARR_COMM_DEF_MAX_PEND_REQ = 64 - integer(SHR_KIND_IN), dimension(NUM_REARR_COMM_OPTS) :: buf - integer :: rank, ierr - - call mpi_comm_rank(comm, rank, ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_rank comm_world') - - buf = 0 - ! buf(1) = comm_type - ! buf(2) = comm_fcd - ! buf(3) = max_pend_req_comp2io - ! buf(4) = enable_hs_comp2io - ! buf(5) = enable_isend_comp2io - ! buf(6) = max_pend_req_io2comp - ! buf(7) = enable_hs_io2comp - ! buf(8) = enable_isend_io2comp - if(rank == 0) then - ! buf(1) = comm_type - select case(pio_rearr_comm_type) - case ("p2p") - case ("default") - buf(1) = pio_rearr_comm_p2p - case ("coll") - buf(1) = pio_rearr_comm_coll - case default - write(shr_log_unit,*) "Invalid PIO rearranger comm type, ", pio_rearr_comm_type - write(shr_log_unit,*) "Resetting PIO rearrange comm type to p2p" - buf(1) = pio_rearr_comm_p2p - end select - - ! buf(2) = comm_fcd - select case(pio_rearr_comm_fcd) - case ("2denable") - case ("default") - buf(2) = pio_rearr_comm_fc_2d_enable - case ("io2comp") - buf(2) = pio_rearr_comm_fc_1d_io2comp - case ("comp2io") - buf(2) = pio_rearr_comm_fc_1d_comp2io - case ("disable") - buf(2) = pio_rearr_comm_fc_2d_disable - case default - write(shr_log_unit,*) "Invalid PIO rearranger comm flow control direction, ", pio_rearr_comm_fcd - write(shr_log_unit,*) "Resetting PIO rearrange comm flow control direction to 2denable" - buf(2) = pio_rearr_comm_fc_2d_enable - end select - - ! buf(3) = max_pend_req_comp2io - if((pio_rearr_comm_max_pend_req_comp2io <= 0) .and. & - (pio_rearr_comm_max_pend_req_comp2io /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then - - ! Small multiple of pio_numiotasks has proven to perform - ! well empirically, and we do not want to allow maximum for - ! very large process count runs. Can improve this by - ! communicating between iotasks first, and then non-iotasks - ! to iotasks (TO DO) - write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (comp2io), ",& - pio_rearr_comm_max_pend_req_comp2io - write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (comp2io) to ", & - max(PIO_REARR_COMM_DEF_MAX_PEND_REQ, 2 * pio_numiotasks) - buf(3) = max(PIO_REARR_COMM_DEF_MAX_PEND_REQ, 2 * pio_numiotasks) - else - buf(3) = pio_rearr_comm_max_pend_req_comp2io - end if - - ! buf(4) = enable_hs_comp2io - if(pio_rearr_comm_enable_hs_comp2io) then - buf(4) = 1 - else - buf(4) = 0 - end if - - ! buf(5) = enable_isend_comp2io - if(pio_rearr_comm_enable_isend_comp2io) then - buf(5) = 1 - else - buf(5) = 0 - end if - - ! buf(6) = max_pend_req_io2comp - if((pio_rearr_comm_max_pend_req_io2comp <= 0) .and. & - (pio_rearr_comm_max_pend_req_io2comp /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then - write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (io2comp), ", pio_rearr_comm_max_pend_req_io2comp - write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (io2comp) to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ - buf(6) = PIO_REARR_COMM_DEF_MAX_PEND_REQ - else - buf(6) = pio_rearr_comm_max_pend_req_io2comp - end if - - ! buf(7) = enable_hs_io2comp - if(pio_rearr_comm_enable_hs_io2comp) then - buf(7) = 1 - else - buf(7) = 0 - end if - - ! buf(8) = enable_isend_io2comp - if(pio_rearr_comm_enable_isend_io2comp) then - buf(8) = 1 - else - buf(8) = 0 - end if - - ! Log the rearranger options - write(shr_log_unit, *) "PIO rearranger options:" - write(shr_log_unit, *) " comm type =", pio_rearr_comm_type - write(shr_log_unit, *) " comm fcd =", pio_rearr_comm_fcd - write(shr_log_unit, *) " max pend req (comp2io) =", pio_rearr_comm_max_pend_req_comp2io - write(shr_log_unit, *) " enable_hs (comp2io) =", pio_rearr_comm_enable_hs_comp2io - write(shr_log_unit, *) " enable_isend (comp2io) =", pio_rearr_comm_enable_isend_comp2io - write(shr_log_unit, *) " max pend req (io2comp) =", pio_rearr_comm_max_pend_req_io2comp - write(shr_log_unit, *) " enable_hs (io2comp) =", pio_rearr_comm_enable_hs_io2comp - write(shr_log_unit, *) " enable_isend (io2comp) =", pio_rearr_comm_enable_isend_io2comp - end if - - call shr_mpi_bcast(buf, comm) - - ! buf(1) = comm_type - ! buf(2) = comm_fcd - ! buf(3) = max_pend_req_comp2io - ! buf(4) = enable_hs_comp2io - ! buf(5) = enable_isend_comp2io - ! buf(6) = max_pend_req_io2comp - ! buf(7) = enable_hs_io2comp - ! buf(8) = enable_isend_io2comp - pio_rearr_opt_comm_type = buf(1) - pio_rearr_opt_fcd = buf(2) - pio_rearr_opt_c2i_max_pend_req = buf(3) - if(buf(4) == 0) then - pio_rearr_opt_c2i_enable_hs = .false. - else - pio_rearr_opt_c2i_enable_hs = .true. - end if - if(buf(5) == 0) then - pio_rearr_opt_c2i_enable_isend = .false. - else - pio_rearr_opt_c2i_enable_isend = .true. - end if - pio_rearr_opt_i2c_max_pend_req = buf(6) - if(buf(7) == 0) then - pio_rearr_opt_i2c_enable_hs = .false. - else - pio_rearr_opt_i2c_enable_hs = .true. - end if - if(buf(8) == 0) then - pio_rearr_opt_i2c_enable_isend = .false. - else - pio_rearr_opt_i2c_enable_isend = .true. - end if - end subroutine -!=============================================================================== - -end module shr_pio_mod From 77705137e8230b00a6eb132b693d8a92af466872 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Thu, 18 Feb 2021 14:49:13 -0600 Subject: [PATCH 09/42] add missing ESMF call to use statement --- mediator/med_io_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 8e69fec60..4953001d9 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -9,7 +9,7 @@ module med_io_mod use shr_const_mod , only : fillvalue => SHR_CONST_SPVAL use ESMF , only : ESMF_VM, ESMF_LogWrite, ESMF_LOGMSG_INFO use ESMF , only : ESMF_SUCCESS, ESMF_FAILURE - use ESMF , only : ESMF_VMGetCurrent, ESMF_VMGet, ESMF_VMBroadCast + use ESMF , only : ESMF_VMGetCurrent, ESMF_VMGet, ESMF_VMBroadCast, ESMF_GridComp use NUOPC , only : NUOPC_FieldDictionaryGetEntry use NUOPC , only : NUOPC_FieldDictionaryHasEntry use pio , only : file_desc_t, iosystem_desc_t From 6e36233f61514f8f79e371dd2b24b1bfa01c5bd7 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Thu, 18 Feb 2021 16:26:45 -0600 Subject: [PATCH 10/42] remove shr_pio_mod.F90 from util/CMakeLists.txt --- util/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt index e99cfda83..6b95eea08 100644 --- a/util/CMakeLists.txt +++ b/util/CMakeLists.txt @@ -2,6 +2,6 @@ project(CMEPS_share Fortran) include(ExternalProject) add_library(cmeps_share shr_abort_mod.F90 shr_flux_mod.F90 shr_log_mod.F90 shr_mpi_mod.F90 shr_sys_mod.F90 - glc_elevclass_mod.F90 perf_mod.F90 shr_const_mod.F90 shr_kind_mod.F90 shr_mem_mod.F90 shr_pio_mod.F90) + glc_elevclass_mod.F90 perf_mod.F90 shr_const_mod.F90 shr_kind_mod.F90 shr_mem_mod.F90) target_include_directories (cmeps_share PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${ESMF_F90COMPILEPATHS} ${PIO_Fortran_INCLUDE_DIRS}) From 42c82d7da1830d14d20447cc9c3675e18b8d920b Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 23 Feb 2021 00:36:09 -0600 Subject: [PATCH 11/42] fix for performance issue in S2S --- mediator/med.F90 | 3 ++- mediator/med_io_mod.F90 | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/mediator/med.F90 b/mediator/med.F90 index 493fac3ca..80c64ddf9 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -2389,7 +2389,8 @@ subroutine DataInitialize(gcomp, rc) !--------------------------------------- ! Initialize mediator IO !--------------------------------------- - call med_io_init(gcomp) + call med_io_init(gcomp, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return !--------------------------------------- ! Initialize mediator water/heat budget diags diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 4953001d9..1948c8161 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -119,7 +119,7 @@ logical function med_io_file_exists(vm, iam, filename) end function med_io_file_exists !=============================================================================== - subroutine med_io_init(gcomp) + subroutine med_io_init(gcomp, rc) !--------------- ! initialize pio @@ -141,11 +141,11 @@ subroutine med_io_init(gcomp) ! input/output arguments type(ESMF_GridComp), intent(in) :: gcomp + integer , intent(out) :: rc #ifndef CESMCOUPLED ! local variables type(ESMF_VM) :: vm - integer :: rc integer :: ret integer :: comm integer :: localPet, petCount @@ -238,6 +238,7 @@ subroutine med_io_init(gcomp) else pio_root = -99 end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_root = ', pio_root ! pio_stride call NUOPC_CompAttributeGet(gcomp, name='pio_stride', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) @@ -247,6 +248,7 @@ subroutine med_io_init(gcomp) else pio_stride = -99 end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride ! pio_numiotasks call NUOPC_CompAttributeGet(gcomp, name='pio_numiotasks', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) @@ -256,12 +258,18 @@ subroutine med_io_init(gcomp) else pio_numiotasks = -99 end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks ! check for parallel IO, it requires at least two io pes if (petCount > 1 .and. pio_numiotasks == 1 .and. & (pio_iotype .eq. PIO_IOTYPE_PNETCDF .or. pio_iotype .eq. PIO_IOTYPE_NETCDF4P)) then pio_numiotasks = 2 pio_stride = min(pio_stride, petCount/2) + if (localPet == 0) then + write(logunit,*) ' parallel io requires at least two io pes - following parameters are updated:' + write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride + write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks + end if endif ! check/set/correct io pio parameters @@ -273,7 +281,7 @@ subroutine med_io_init(gcomp) pio_stride = max(1,petCount/4) pio_numiotasks = max(1,petCount/pio_stride) end if - if(pio_stride == 1) then + if (pio_stride == 1) then pio_root = 0 endif @@ -294,8 +302,10 @@ subroutine med_io_init(gcomp) pio_root = 0 end if if (localPet == 0) then - write(logunit,*) 'pio_stride, iotasks or root out of bounds - resetting to defaults: ', & - pio_stride, pio_numiotasks, pio_root + write(logunit,*) 'pio_stride, iotasks or root out of bounds - resetting to defaults:' + write(logunit,*) trim(subname), ' : pio_root = ', pio_root + write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride + write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks end if end if @@ -314,8 +324,8 @@ subroutine med_io_init(gcomp) return end if else - cvalue = 'SUBSET' - pio_rearranger = PIO_REARR_SUBSET + cvalue = 'BOX' + pio_rearranger = PIO_REARR_BOX end if if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearranger = ', trim(cvalue), pio_rearranger From 21b2d8629cdef17e676255db9da7fd12948a6cdb Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 23 Feb 2021 00:41:25 -0600 Subject: [PATCH 12/42] change default for pio_root --- mediator/med_io_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 1948c8161..8eb0abdb0 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -236,7 +236,7 @@ subroutine med_io_init(gcomp, rc) endif pio_root = min(pio_root, petCount-1) else - pio_root = -99 + pio_root = 1 end if if (localPet == 0) write(logunit,*) trim(subname), ' : pio_root = ', pio_root From dced80b53a35463712f2a5e58f3c09dd36d032a0 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 23 Feb 2021 01:06:09 -0600 Subject: [PATCH 13/42] remove extra print --- mediator/med_io_mod.F90 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 8eb0abdb0..8f10cbb13 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -329,13 +329,6 @@ subroutine med_io_init(gcomp, rc) end if if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearranger = ', trim(cvalue), pio_rearranger - ! print out PIO init parameters - if (localPet == 0) then - write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks - write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride - write(logunit,*) trim(subname), ' : pio_root = ', pio_root - end if - ! init PIO allocate(io_subsystem) if (localPet == 0) write(logunit,*) trim(subname),' calling pio init' From 48b9136b991c10a439968a1aadd3bb03d1203d81 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 23 Feb 2021 01:58:13 -0600 Subject: [PATCH 14/42] add extra print --- mediator/med_io_mod.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index 8f10cbb13..dc7e129f7 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -275,11 +275,15 @@ subroutine med_io_init(gcomp, rc) ! check/set/correct io pio parameters if (pio_stride > 0 .and. pio_numiotasks < 0) then pio_numiotasks = max(1, petCount/pio_stride) + if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_numiotasks = ', pio_numiotasks else if(pio_numiotasks > 0 .and. pio_stride < 0) then pio_stride = max(1, petCount/pio_numiotasks) + if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_stride = ', pio_stride else if(pio_numiotasks < 0 .and. pio_stride < 0) then pio_stride = max(1,petCount/4) pio_numiotasks = max(1,petCount/pio_stride) + if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_numiotasks = ', pio_numiotasks + if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_stride = ', pio_stride end if if (pio_stride == 1) then pio_root = 0 From 216ab10d6dde89330ed33ad25208554de9730d77 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 26 Feb 2021 12:04:28 -0600 Subject: [PATCH 15/42] add support to set pio debug level and set uninitialized variable --- mediator/med_io_mod.F90 | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index dc7e129f7..08ff3bac9 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -107,6 +107,7 @@ logical function med_io_file_exists(vm, iam, filename) integer :: rc !------------------------------------------------------------------------------- + tmp(1) = 0 med_io_file_exists = .false. if (iam==0) inquire(file=trim(filename),exist=med_io_file_exists) if (med_io_file_exists) tmp(1) = 1 @@ -128,7 +129,7 @@ subroutine med_io_init(gcomp, rc) #ifdef CESMCOUPLED use shr_pio_mod , only : shr_pio_getiosys, shr_pio_getiotype, shr_pio_getioformat #else - use pio , only : pio_init, pio_set_rearr_opts + use pio , only : pio_init, pio_setdebuglevel, pio_set_rearr_opts use pio , only : PIO_64BIT_OFFSET, PIO_64BIT_DATA use pio , only : PIO_IOTYPE_NETCDF, PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P use pio , only : PIO_REARR_BOX, PIO_REARR_SUBSET @@ -153,6 +154,7 @@ subroutine med_io_init(gcomp, rc) integer :: pio_stride integer :: pio_rearranger integer :: pio_root + integer :: pio_debug_level integer :: pio_rearr_comm_type integer :: pio_rearr_comm_fcd logical :: pio_rearr_comm_enable_hs_comp2io @@ -183,6 +185,7 @@ subroutine med_io_init(gcomp, rc) ! pio_netcdf_format call NUOPC_CompAttributeGet(gcomp, name='pio_netcdf_format', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then cvalue = ESMF_UtilStringUpperCase(cvalue) if (trim(cvalue) .eq. 'CLASSIC') then @@ -205,6 +208,7 @@ subroutine med_io_init(gcomp, rc) ! pio_typename call NUOPC_CompAttributeGet(gcomp, name='pio_typename', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then cvalue = ESMF_UtilStringUpperCase(cvalue) if (trim(cvalue) .eq. 'NETCDF') then @@ -229,6 +233,7 @@ subroutine med_io_init(gcomp, rc) ! pio_root call NUOPC_CompAttributeGet(gcomp, name='pio_root', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then read(cvalue,*) pio_root if (pio_root < 0) then @@ -243,6 +248,7 @@ subroutine med_io_init(gcomp, rc) ! pio_stride call NUOPC_CompAttributeGet(gcomp, name='pio_stride', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then read(cvalue,*) pio_stride else @@ -253,6 +259,7 @@ subroutine med_io_init(gcomp, rc) ! pio_numiotasks call NUOPC_CompAttributeGet(gcomp, name='pio_numiotasks', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then read(cvalue,*) pio_numiotasks else @@ -316,6 +323,7 @@ subroutine med_io_init(gcomp, rc) ! pio_rearranger call NUOPC_CompAttributeGet(gcomp, name='pio_rearranger', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then cvalue = ESMF_UtilStringUpperCase(cvalue) if (trim(cvalue) .eq. 'BOX') then @@ -338,6 +346,26 @@ subroutine med_io_init(gcomp, rc) if (localPet == 0) write(logunit,*) trim(subname),' calling pio init' call pio_init(localPet, comm, pio_numiotasks, 0, pio_stride, pio_rearranger, io_subsystem, base=pio_root) + ! PIO debug related options + ! pio_debug_level + call NUOPC_CompAttributeGet(gcomp, name='pio_debug_level', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + read(cvalue,*) pio_debug_level + if (pio_debug_level < 0 .or. pio_debug_level > 6) then + call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_debug_level (0-6)', ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + pio_debug_level = 0 + end if + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_debug_level = ', pio_debug_level + + ! set PIO debug level + call pio_setdebuglevel(pio_debug_level) + ! query shared PIO rearranger attributes ! pio_rearr_comm_type call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_type', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) From 3f9de915783d8a740bba21ebc7dad1bca0b24eee Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 26 Feb 2021 12:16:53 -0600 Subject: [PATCH 16/42] minor fix for CDEPS data ocean --- mediator/med.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mediator/med.F90 b/mediator/med.F90 index 80c64ddf9..ecc17851d 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -2036,13 +2036,13 @@ subroutine DataInitialize(gcomp, rc) if ( is_local%wrap%med_coupling_active(compocn,compatm) .or. is_local%wrap%med_coupling_active(compatm,compocn)) then - if (.not. is_local%wrap%med_coupling_active(compatm,compocn)) then - is_local%wrap%med_coupling_active(compatm,compocn) = .true. - end if - ! Create field bundles for mediator ocean albedo computation fieldCount = med_fldList_GetNumFlds(fldListMed_ocnalb) if (fieldCount > 0) then + if (.not. is_local%wrap%med_coupling_active(compatm,compocn)) then + is_local%wrap%med_coupling_active(compatm,compocn) = .true. + end if + allocate(fldnames(fieldCount)) call med_fldList_getfldnames(fldListMed_ocnalb%flds, fldnames, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return From d2450941fc07067a9ab651688a097e98a8e58da9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 26 Feb 2021 12:50:28 -0700 Subject: [PATCH 17/42] use pts_lat and pts_lon for single point cases --- cime_config/namelist_definition_drv.xml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 5d146780b..7e130c695 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -720,11 +720,10 @@ ALLCOMP_attributes grid point latitude associated with single column mode. - if set to -999, ignore this value + used for single column and single point grids - -999. - $PTS_LAT + $PTS_LAT @@ -737,8 +736,7 @@ set by PTS_LON in env_run.xml. - -999. - $PTS_LON + $PTS_LON @@ -1095,7 +1093,7 @@ mapping LND_attributes - DOMAIN description of lnd grid - this will be depracated + DOMAIN description of lnd grid - this will be depracated only here for backwards compatibility From 8b29011564f6170afac874abc6c70364e9d69975 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 1 Mar 2021 09:49:05 -0700 Subject: [PATCH 18/42] remove PTS_MODE variable, use domain file --- cime_config/buildnml | 3 ++- cime_config/config_component.xml | 15 ++++----------- cime_config/namelist_definition_drv.xml | 12 ++++++------ 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index d7fc2dc0e..e7c79d142 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -43,7 +43,8 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['MPILIB'] = case.get_value('MPILIB') config['OS'] = case.get_value('OS') config['glc_nec'] = 0 if case.get_value('GLC_NEC') == 0 else case.get_value('GLC_NEC') - config['single_column'] = 'true' if case.get_value('PTS_MODE') else 'false' + if case.get_value("ATM_NX") == 1 and case.get_value("ATM_NY") == 1 and case.get_value("ATM_DOMAIN_FILE") != "UNSET": + config['single_column'] = 'true' config['timer_level'] = 'pos' if case.get_value('TIMER_LEVEL') >= 1 else 'neg' config['bfbflag'] = 'on' if case.get_value('BFBFLAG') else 'off' config['continue_run'] = '.true.' if case.get_value('CONTINUE_RUN') else '.false.' diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index facba433d..c8593f667 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1233,21 +1233,13 @@ grid mask - DO NOT EDIT (for experts only) - - logical - TRUE,FALSE - FALSE - run_domain - env_run.xml - Operate on only a single point of the global grid - DO NOT EDIT (for experts only) - - real -999.99 run_domain env_run.xml - Latitude to find nearest points for points mode (only used if PTS_MODE is TRUE) + Latitude of grid location, in single column mode interpolate datasets to this location + in single point mode assume all datasets are at this location @@ -1255,7 +1247,8 @@ -999.99 run_domain env_run.xml - Longitude to find nearest points for points mode (only used if PTS_MODE is TRUE) + Longitude of grid location, in single column mode interpolate datasets to this location + in single point mode assume all datasets are at this location diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 7e130c695..783db1e0f 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -701,12 +701,12 @@ - + logical single_column ALLCOMP_attributes - turns on single column mode. set by PTS_MODE in env_case.xml, default: false + Turns on single column mode. .false. @@ -719,8 +719,8 @@ single_column ALLCOMP_attributes - grid point latitude associated with single column mode. - used for single column and single point grids + Latitude in degrees corresponding to the grid location + in single column and single point modes. $PTS_LAT @@ -732,8 +732,8 @@ single_column ALLCOMP_attributes - grid point longitude associated with single column mode. - set by PTS_LON in env_run.xml. + Longitude in degrees corresponding to the grid location + in single column and single point modes. $PTS_LON From b7e7806ecd721a27f4fd75ae1963f43e96e23259 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Mon, 8 Mar 2021 21:51:34 -0700 Subject: [PATCH 19/42] changes for single column mode --- cime_config/buildnml | 10 +- cime_config/config_component.xml | 173 ++++++++++++------------ cime_config/namelist_definition_drv.xml | 66 ++++----- drivers/cime/esm.F90 | 33 ----- 4 files changed, 122 insertions(+), 160 deletions(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index e7c79d142..6f637ba6e 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -43,8 +43,6 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['MPILIB'] = case.get_value('MPILIB') config['OS'] = case.get_value('OS') config['glc_nec'] = 0 if case.get_value('GLC_NEC') == 0 else case.get_value('GLC_NEC') - if case.get_value("ATM_NX") == 1 and case.get_value("ATM_NY") == 1 and case.get_value("ATM_DOMAIN_FILE") != "UNSET": - config['single_column'] = 'true' config['timer_level'] = 'pos' if case.get_value('TIMER_LEVEL') >= 1 else 'neg' config['bfbflag'] = 'on' if case.get_value('BFBFLAG') else 'off' config['continue_run'] = '.true.' if case.get_value('CONTINUE_RUN') else '.false.' @@ -52,6 +50,14 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['atm_grid'] = case.get_value('ATM_GRID') config['mask_grid'] = case.get_value('MASK_GRID') + # determine if need to set atm_domainfile + scol_lon = float(case.get_value('PTS_LON')) + scol_lat = float(case.get_value('PTS_LAT')) + if scol_lon >= 0. and scol_lat >= 0. and case.get_value("ATM_DOMAIN_FILE") != "UNSET": + config['single_column'] = 'true' + else: + config['single_column'] = 'false' + # needed for determining the run sequence as well as glc_renormalize_smb config['COMP_ATM'] = case.get_value("COMP_ATM") config['COMP_ICE'] = case.get_value("COMP_ICE") diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index c8593f667..e14af5aab 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1251,24 +1251,30 @@ in single point mode assume all datasets are at this location - - - - - + char UNSET run_domain env_run.xml - atm domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of atm domain file + used only if if PTS_LAT and PTS_LON are greater than or + equal to 0. If this is the case then if PTS_DOMAINFILE is not + equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using + PTS_LAT and PTS_LON will be done and the component mesh will have + this nearest neighbor value. + + + + + + + + + + + + + char UNSET @@ -1291,6 +1297,7 @@ env_run.xml path of lnd domain file + char UNSET @@ -1299,20 +1306,20 @@ lnd mesh file (full pathname) - - char - UNSET - run_domain - env_run.xml - rof domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of rof domain file - + + + + + + + + + + + + + + char UNSET @@ -1321,20 +1328,20 @@ rof mesh file (full pathname) - - char - UNSET - run_domain - env_run.xml - wav domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of wav domain file - + + + + + + + + + + + + + + char UNSET @@ -1343,20 +1350,20 @@ wav mesh file (full pathname) - - char - UNSET - run_domain - env_run.xml - ice domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of ice domain file - + + + + + + + + + + + + + + char UNSET @@ -1365,20 +1372,20 @@ ice mesh file (full pahtname) - - char - UNSET - run_domain - env_run.xml - ocn domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of ocn domain file - + + + + + + + + + + + + + + char UNSET @@ -1387,20 +1394,20 @@ ocn mesh file (full pathname) - - char - UNSET - run_domain - env_run.xml - glc domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of glc domain file - + + + + + + + + + + + + + + char UNSET diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 783db1e0f..d14e3e245 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -701,42 +701,44 @@ - - logical + + real single_column ALLCOMP_attributes - Turns on single column mode. + Latitude in degrees corresponding to the grid location + in single column and single point modes. - .false. - .true. + $PTS_LAT - + real single_column ALLCOMP_attributes - Latitude in degrees corresponding to the grid location + Longitude in degrees corresponding to the grid location in single column and single point modes. - $PTS_LAT + $PTS_LON - - real + + char single_column ALLCOMP_attributes - Longitude in degrees corresponding to the grid location - in single column and single point modes. + DOMAIN file needed for single column model IF and only if + a nearest neighbor search is done to match the PTS_LAT and PTS_LON + to the closest point in the domain file. This file is ONLY used in + single column mode. - $PTS_LON + $PTS_DOMAINFILE @@ -1070,38 +1072,11 @@ $MASK_MESH + null - - char - mapping - ALLCOMP_attributes - - DOMAIN file needed for single column model - A nearest neighbor search is done to match the PTS_LAT and PTS_LON - to the closest point in the domain file. This file is ONLY used in - single column mode. - - - $ATM_DOMAIN_PATH/$ATM_DOMAIN_FILE - - - - - char - mapping - LND_attributes - - DOMAIN description of lnd grid - this will be depracated - only here for backwards compatibility - - - $LND_DOMAIN_PATH/$LND_DOMAIN_FILE - - - - + char mapping ATM_attributes @@ -1110,6 +1085,7 @@ $ATM_DOMAIN_MESH + null @@ -1122,6 +1098,7 @@ $LND_DOMAIN_MESH + null @@ -1134,6 +1111,7 @@ $OCN_DOMAIN_MESH + null @@ -1146,6 +1124,7 @@ $ICE_DOMAIN_MESH + null @@ -1158,6 +1137,7 @@ $ROF_DOMAIN_MESH + null @@ -1170,6 +1150,7 @@ $GLC_DOMAIN_MESH + null @@ -1182,6 +1163,7 @@ $WAV_DOMAIN_MESH + null diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index 821e7e8de..652abea13 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -429,9 +429,6 @@ subroutine InitAttributes(driver, rc) character(LEN=CS) :: tfreeze_option ! Freezing point calculation real(R8) :: wall_time_limit ! wall time limit in hours integer :: glc_nec ! number of elevation classes in the land component for lnd->glc - logical :: single_column ! scm mode logical - real(R8) :: scmlon ! single column lon - real(R8) :: scmlat ! single column lat character(LEN=CS) :: wv_sat_scheme real(R8) :: wv_sat_transition_start logical :: wv_sat_use_tables @@ -483,7 +480,6 @@ subroutine InitAttributes(driver, rc) call shr_frz_freezetemp_init(tfreeze_option, mastertask) - call NUOPC_CompAttributeGet(driver, name='cpl_rootpe', value=cvalue, rc=rc) read(cvalue, *) rootpe_med if (chkerr(rc,__LINE__,u_FILE_u)) return @@ -495,11 +491,9 @@ subroutine InitAttributes(driver, rc) !---------------------------------------------------------- ! This must be called on all processors of the driver - call NUOPC_CompAttributeGet(driver, name='glc_nec', value=cvalue, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return read(cvalue,*) glc_nec - call glc_elevclass_init(glc_nec, logunit=logunit) !---------------------------------------------------------- @@ -561,33 +555,6 @@ subroutine InitAttributes(driver, rc) call shr_wv_sat_make_tables(liquid_spec, ice_spec, mixed_spec) end if - !---------------------------------------------------------- - ! Set single_column flags - ! If in single column mode, overwrite flags according to focndomain file - ! in ocn_in namelist. SCAM can reset the "present" flags for lnd, - ! ocn, ice, rof, and flood. - !---------------------------------------------------------- - - call NUOPC_CompAttributeGet(driver, name="single_column", value=cvalue, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - read(cvalue,*) single_column - - ! NOTE: cam stand-alone aqua-planet model will no longer be supported here - only the data model aqua-planet - ! will be supported - if (single_column) then - - call NUOPC_CompAttributeGet(driver, name="scmlon", value=cvalue, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - read(cvalue,*) scmlon - - call NUOPC_CompAttributeGet(driver, name="scmlat", value=cvalue, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - read(cvalue,*) scmlat - - ! TODO(mvertens, 2019-01-30): need to add single column functionality - - endif - end subroutine InitAttributes !================================================================================ From 6ffb9f168276dc2e9ae56bd28cd3877b7cf72333 Mon Sep 17 00:00:00 2001 From: "Bin.Li" Date: Tue, 9 Mar 2021 17:04:51 +0000 Subject: [PATCH 20/42] Updated land-sea mask used for CFSR or GEFS data. --- mediator/med_map_mod.F90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index c7e10469f..475fb5232 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -258,6 +258,9 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, if (n1 == compatm .and. (n2 == compocn .or. n2 == compice)) then srcMaskValue = 1 dstMaskValue = 0 + if (atm_name(1:9) == 'datm_cfsr' .or. atm_name(1:9) == 'datm_gefs' ) then + srcMaskValue = 0 + endif else if (n2 == compatm .and. (n1 == compocn .or. n1 == compice)) then srcMaskValue = 0 dstMaskValue = 1 From 0dccdad84184d0d63aab6ac4a6383ff7b997536c Mon Sep 17 00:00:00 2001 From: mvertens Date: Fri, 12 Mar 2021 12:33:57 -0700 Subject: [PATCH 21/42] changes to get scol working --- cime_config/buildnml | 2 +- cime_config/namelist_definition_drv.xml | 2 +- drivers/cime/esm.F90 | 285 +++++++++++++++++++++++- 3 files changed, 282 insertions(+), 7 deletions(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index 6f637ba6e..af6ba9011 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -53,7 +53,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # determine if need to set atm_domainfile scol_lon = float(case.get_value('PTS_LON')) scol_lat = float(case.get_value('PTS_LAT')) - if scol_lon >= 0. and scol_lat >= 0. and case.get_value("ATM_DOMAIN_FILE") != "UNSET": + if scol_lon > -999. and scol_lat > -999. and case.get_value("ATM_DOMAIN_FILE") != "UNSET": config['single_column'] = 'true' else: config['single_column'] = 'false' diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index d14e3e245..249683e8e 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -727,7 +727,7 @@ - + char single_column ALLCOMP_attributes diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index 652abea13..09e0c1eed 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -30,6 +30,8 @@ module ESM private :: esm_finalize private :: pretty_print_nuopc_freeformat + real(r8) :: scol_spval = -999._r8 + character(*), parameter :: u_FILE_u = & __FILE__ @@ -660,7 +662,7 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, r call NUOPC_CompAttributeGet(driver, name="mediator_read_restart", value=cvalue, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return read(cvalue,*) lvalue - if (.not. lvalue) then + if (.not. lvalue) then call NUOPC_CompAttributeGet(driver, name=trim(attribute), value=cvalue, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return end if @@ -711,6 +713,13 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, r if (chkerr(rc,__LINE__,u_FILE_u)) return end if + !------ + ! Add single column and single point attributes + !------ + + call esm_set_single_column_attributes(compname, gcomp, rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + end subroutine AddAttributes !================================================================================ @@ -854,7 +863,7 @@ subroutine esm_init_pelayout(driver, maxthreads, rc) logical, allocatable :: comp_iamin(:) character(len=5) :: inst_suffix character(CL) :: cvalue - logical :: found_comp + logical :: found_comp character(len=*), parameter :: subname = "(esm_pelayout.F90:esm_init_pelayout)" !--------------------------------------- @@ -1029,9 +1038,6 @@ subroutine esm_init_pelayout(driver, maxthreads, rc) return endif - call AddAttributes(child, driver, config, i+1, trim(compLabels(i)), inst_suffix, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - if (ESMF_GridCompIsPetLocal(child, rc=rc)) then call ESMF_GridCompGet(child, vm=vm, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return @@ -1066,6 +1072,274 @@ end subroutine esm_init_pelayout !================================================================================ + subroutine esm_set_single_column_attributes(compname, gcomp, rc) + + ! Generate a mesh for single column + + use netcdf + use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd + use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet, ESMF_SUCCESS + + ! input/output variables + character(len=*) , intent(in) :: compname + type(ESMF_GridComp) , intent(inout) :: gcomp + integer , intent(out) :: rc + + ! local variables + type(ESMF_VM) :: vm + character(len=CL) :: single_column_lnd_domainfile + real(r8) :: scol_lon + real(r8) :: scol_lat + real(r8) :: scol_area + integer :: scol_lndmask + real(r8) :: scol_lndfrac + integer :: scol_ocnmask + real(r8) :: scol_ocnfrac + integer :: i,j,ni,nj + integer :: ncid + integer :: dimid + integer :: varid_xc + integer :: varid_yc + integer :: varid_area + integer :: varid_mask + integer :: varid_frac + integer :: start(2) ! Start index to read in + integer :: start3(3) ! Start index to read in + integer :: count3(3) ! Number of points to read in + integer :: status ! status flag + real (r8), allocatable :: lats(:) ! temporary + real (r8), allocatable :: lons(:) ! temporary + real (r8), allocatable :: pos_lons(:) ! temporary + real (r8), allocatable :: glob_grid(:,:) ! temporary + real (r8) :: pos_scol_lon ! temporary + real (r8) :: scol_data(1) + integer :: iscol_data(1) + integer :: petcount + character(len=CL) :: cvalue + character(len=*), parameter :: subname= ' (esm_get_single_column_attributes) ' + !------------------------------------------------------------------------------- + + + rc = ESMF_SUCCESS + + ! obtain the single column lon and lat + call NUOPC_CompAttributeGet(gcomp, name='scol_lon', value=cvalue, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + read(cvalue,*) scol_lon + call NUOPC_CompAttributeGet(gcomp, name='scol_lat', value=cvalue, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + read(cvalue,*) scol_lat + call NUOPC_CompAttributeGet(gcomp, name='single_column_lnd_domainfile', value=single_column_lnd_domainfile, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if ( (scol_lon < scol_spval .and. scol_lat > scol_spval) .or. & + (scol_lon > scol_spval .and. scol_lat < scol_spval)) then + call shr_sys_abort(subname//' ERROR: '//trim(compname)//' both scol_lon and scol_lat must be greater than -999 ') + end if + + if (scol_lon > -999. .and. scol_lat > -999.) then + + ! NOTE: currently assume that single column capability is restricted to + ! ATM, OCN and ICE components only + ! verify that ROF, WAV and LND are not trying to use single column mode + if (trim(compname) == 'ROF' .or. trim(compname) == 'WAV') then + call shr_sys_abort(subname//' ERROR: '//trim(compname)//' does not support single column mode ') + end if + + ! ensure that single column mode is only run on 1 pet + call ESMF_GridCompGet(gcomp, vm=vm, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_VMGet(vm, petcount=petcount, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (petcount > 1) then + call shr_sys_abort(subname//' ERROR: single column mode must be run on 1 pe') + endif + + write(logunit,'(a,2(f10.5,2x))')trim(subname)//' single column point for '//trim(compname)//' has lon and lat = ',& + scol_lon,scol_lat + + ! This is either a single column or a single point so add attributes + call NUOPC_CompAttributeAdd(gcomp, & + attrList=(/'scol_area ', & + 'scol_ni ', & + 'scol_nj ', & + 'scol_lndmask', & + 'scol_lndfrac', & + 'scol_ocnmask', & + 'scol_ocnfrac'/), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + if (trim(single_column_lnd_domainfile) /= 'null') then + + ! In this case the domain file is not a single point file - but normally a + ! global domain file where a nearest neighbor search will be done to find + ! the closest point in the domin file to scol_lon and scol_lat + + status = nf90_open(single_column_lnd_domainfile, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) then + call shr_sys_abort (trim(subname) //': Cannot open '//trim(single_column_lnd_domainfile)) + endif + status = nf90_inq_dimid (ncid, 'ni', dimid) + status = nf90_inquire_dimension(ncid, dimid, len=ni) + status = nf90_inq_dimid (ncid, 'nj', dimid) + status = nf90_inquire_dimension(ncid, dimid, len=nj) + + status = nf90_open(single_column_lnd_domainfile, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) then + call shr_sys_abort (trim(subname) //': Cannot open '//trim(single_column_lnd_domainfile)) + endif + status = nf90_inq_dimid (ncid, 'ni', dimid) + status = nf90_inquire_dimension(ncid, dimid, len=ni) + status = nf90_inq_dimid (ncid, 'nj', dimid) + status = nf90_inquire_dimension(ncid, dimid, len=nj) + + status = nf90_inq_varid(ncid, 'xc' , varid_xc) + if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid xc') + status = nf90_inq_varid(ncid, 'yc' , varid_yc) + if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid yc') + status = nf90_inq_varid(ncid, 'area' , varid_area) + if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid area') + status = nf90_inq_varid(ncid, 'mask' , varid_mask) + if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid mask') + status = nf90_inq_varid(ncid, 'frac' , varid_frac) + if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid frac') + + ! Read in domain file for single column + allocate(lats(nj)) + allocate(lons(ni)) + allocate(pos_lons(ni)) + allocate(glob_grid(ni,nj)) + + ! The follow assumes that xc and yc are 2 dimensional values + start3=(/1,1,1/) + count3=(/ni,nj,1/) + status = nf90_get_var(ncid, varid_xc, glob_grid, start3, count3) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var xc') + do i = 1,ni + lons(i) = glob_grid(i,1) + end do + status = nf90_get_var(ncid, varid_yc, glob_grid, start3, count3) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var yc') + do j = 1,nj + lats(j) = glob_grid(1,j) + end do + ! find nearest neighbor indices of scol_lon and scol_lat in single_column_lnd_domain file + ! convert lons array and scol_lon to 0,360 and find index of value closest to 0 + ! and obtain single-column longitude/latitude indices to retrieve + pos_lons(:) = mod(lons(:) + 360._r8, 360._r8) + pos_scol_lon = mod(scol_lon + 360._r8, 360._r8) + start(1) = (MINLOC(abs(pos_lons - pos_scol_lon), dim=1)) + start(2) = (MINLOC(abs(lats -scol_lat ), dim=1)) + + deallocate(lats) + deallocate(lons) + deallocate(pos_lons) + deallocate(glob_grid) + + ! read in value of nearest neighbor lon and RESET scol_lon and scol_lat + ! also get area of gridcell, mask and frac + status = nf90_get_var(ncid, varid_xc, scol_lon, start) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var xc') + + status = nf90_get_var(ncid, varid_yc, scol_lat, start) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var yc') + + status = nf90_get_var(ncid, varid_area, scol_area, start) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var area') + + status = nf90_get_var(ncid, varid_mask, iscol_data, start) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var mask') + scol_lndmask = iscol_data(1) + scol_ocnmask = 1 - scol_lndmask + + status = nf90_get_var(ncid, varid_frac, scol_data, start) + if (status /= nf90_noerr) call shr_sys_abort (subname//' get_var frac') + scol_lndfrac = scol_data(1) + scol_ocnfrac = 1._r8 - scol_lndfrac + + if (scol_ocnmask == 0 .and. scol_lndmask == 0) then + call shr_sys_abort(trim(subname)//' in single column mode '& + //' ocean and land mask cannot both be zero') + end if + + write(cvalue,*) scol_lon + call NUOPC_CompAttributeSet(gcomp, name='scol_lon', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) scol_lat + call NUOPC_CompAttributeSet(gcomp, name='scol_lat', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) ni + call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) nj + call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + status = nf90_close(ncid) + + write(logunit,'(a,2(f13.5,2x))')trim(subname)//' nearest neighbor scol_lon and scol_lat in '& + //trim(single_column_lnd_domainfile)//' are ',scol_lon,scol_lat + if (trim(compname) == 'LND') then + write(logunit,'(a,i4,f13.5)')trim(subname)//' scol_lndmask, scol_lndfrac are ',& + scol_lndmask, scol_lndfrac + else if (trim(compname) == 'OCN') then + write(logunit,'(a,i4,f13.5)')trim(subname)//' scol_ocnmask, scol_ocnfrac are ',& + scol_ocnmask, scol_ocnfrac + else + write(logunit,'(a)')trim(subname)//' atm point has unit mask and unit fraction ' + end if + + else + + ! for single point mode - its assumed that this point is always + ! either over all ocean or all land and the point is always valid + + scol_lndmask = 1 + scol_lndfrac = 1._r8 + scol_ocnmask = 1 + scol_ocnfrac = 1._r8 + scol_area = 1.e30 + + call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + write(cvalue,*) 1 + call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + write(cvalue,*) 1 + + write(logunit,'(a)')' single point mode is active' + write(logunit,'(a,f13.5,a,f13.5,a)')' scol_lon is ',scol_lon,' and scol_lat is ' + + end if + + write(cvalue,*) scol_area + call NUOPC_CompAttributeSet(gcomp, name='scol_area', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) scol_lndmask + call NUOPC_CompAttributeSet(gcomp, name='scol_lndmask', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) scol_lndfrac + call NUOPC_CompAttributeSet(gcomp, name='scol_lndfrac', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) scol_ocnmask + call NUOPC_CompAttributeSet(gcomp, name='scol_ocnmask', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + write(cvalue,*) scol_ocnfrac + call NUOPC_CompAttributeSet(gcomp, name='scol_ocnfrac', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + end if + + end subroutine esm_set_single_column_attributes + + !================================================================================ subroutine esm_finalize(driver, rc) use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet @@ -1113,4 +1387,5 @@ subroutine esm_finalize(driver, rc) end subroutine esm_finalize + end module ESM From 6cb045409db67c166bf4adc1b10152083608150f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 10 Mar 2021 09:10:48 -0700 Subject: [PATCH 22/42] changes to get nag 6.2 compiler working --- drivers/cime/esm.F90 | 6 ++++-- drivers/cime/esm_utils_mod.F90 | 14 ++++++++------ mediator/med_phases_post_glc_mod.F90 | 6 +++--- mediator/med_utils_mod.F90 | 14 ++++++++------ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index 821e7e8de..f01b86d0b 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -976,8 +976,10 @@ subroutine esm_init_pelayout(driver, maxthreads, rc) return endif - if (allocated(petlist) .and. size(petlist) .ne. ntasks) then - deallocate(petlist) + if (allocated(petlist)) then + if(size(petlist) .ne. ntasks) then + deallocate(petlist) + endif endif if(.not. allocated(petlist)) then allocate(petlist(ntasks)) diff --git a/drivers/cime/esm_utils_mod.F90 b/drivers/cime/esm_utils_mod.F90 index cb5dc4360..dec3c593a 100644 --- a/drivers/cime/esm_utils_mod.F90 +++ b/drivers/cime/esm_utils_mod.F90 @@ -36,15 +36,17 @@ logical function ChkErr(rc, line, file, mpierr) ChkErr = .false. lrc = rc - if (present(mpierr) .and. mpierr) then - if (rc == MPI_SUCCESS) return + if (present(mpierr)) then + if(mpierr) then + if (rc == MPI_SUCCESS) return #ifdef USE_MPI2 - call MPI_ERROR_STRING(rc, lstring, len, ierr) + call MPI_ERROR_STRING(rc, lstring, len, ierr) #else - write(lstring,*) "ERROR in mct mpi-serial library rc=",rc + write(lstring,*) "ERROR in mct mpi-serial library rc=",rc #endif - call ESMF_LogWrite("ERROR: "//trim(lstring), ESMF_LOGMSG_INFO, line=line, file=file, rc=dbrc) - lrc = ESMF_FAILURE + call ESMF_LogWrite("ERROR: "//trim(lstring), ESMF_LOGMSG_INFO, line=line, file=file, rc=dbrc) + lrc = ESMF_FAILURE + endif endif if (ESMF_LogFoundError(rcToCheck=lrc, msg=ESMF_LOGERR_PASSTHRU, line=line, file=file)) then diff --git a/mediator/med_phases_post_glc_mod.F90 b/mediator/med_phases_post_glc_mod.F90 index 5b7b23f13..e04fc64b4 100644 --- a/mediator/med_phases_post_glc_mod.F90 +++ b/mediator/med_phases_post_glc_mod.F90 @@ -135,9 +135,9 @@ subroutine med_phases_post_glc(gcomp, rc) end if end do if (mastertask) then - write(logunit,'(a,l)') trim(subname) // 'glc2lnd_coupling is ',glc2lnd_coupling - write(logunit,'(a,l)') trim(subname) // 'glc2ocn_coupling is ',glc2ocn_coupling - write(logunit,'(a,l)') trim(subname) // 'glc2ice_coupling is ',glc2ice_coupling + write(logunit,'(a,L1)') trim(subname) // 'glc2lnd_coupling is ',glc2lnd_coupling + write(logunit,'(a,L1)') trim(subname) // 'glc2ocn_coupling is ',glc2ocn_coupling + write(logunit,'(a,L1)') trim(subname) // 'glc2ice_coupling is ',glc2ice_coupling end if ! determine if coupling to CISM is 2-way diff --git a/mediator/med_utils_mod.F90 b/mediator/med_utils_mod.F90 index fc012217b..6c3b59638 100644 --- a/mediator/med_utils_mod.F90 +++ b/mediator/med_utils_mod.F90 @@ -54,15 +54,17 @@ logical function med_utils_ChkErr(rc, line, file, mpierr) med_utils_ChkErr = .false. lrc = rc - if (present(mpierr) .and. mpierr) then - if (rc == MPI_SUCCESS) return + if (present(mpierr)) then + if(mpierr) then + if (rc == MPI_SUCCESS) return #ifdef USE_MPI2 - call MPI_ERROR_STRING(rc, lstring, len, ierr) + call MPI_ERROR_STRING(rc, lstring, len, ierr) #else - write(lstring,*) "ERROR in mct mpi-serial library rc=",rc + write(lstring,*) "ERROR in mct mpi-serial library rc=",rc #endif - call ESMF_LogWrite("ERROR: "//trim(lstring), ESMF_LOGMSG_INFO, line=line, file=file) - lrc = ESMF_FAILURE + call ESMF_LogWrite("ERROR: "//trim(lstring), ESMF_LOGMSG_INFO, line=line, file=file) + lrc = ESMF_FAILURE + endif endif if (ESMF_LogFoundError(rcToCheck=lrc, msg=ESMF_LOGERR_PASSTHRU, line=line, file=file)) then From bd5393674a376d239e80467932552ec485dd1ba1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 11 Mar 2021 15:50:12 -0700 Subject: [PATCH 23/42] initialize rc --- drivers/cime/esm_time_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cime/esm_time_mod.F90 b/drivers/cime/esm_time_mod.F90 index 72b246ccd..55b269cde 100644 --- a/drivers/cime/esm_time_mod.F90 +++ b/drivers/cime/esm_time_mod.F90 @@ -643,6 +643,7 @@ subroutine esm_time_read_restart(restart_file, start_ymd, start_tod, curr_ymd, c !---------------------------------------------------------------- ! use netcdf here since it's serial + rc = ESMF_SUCCESS status = nf90_open(restart_file, NF90_NOWRITE, ncid) if (status /= nf90_NoErr) then call ESMF_LogWrite(trim(subname)//' ERROR: nf90_open: '//trim(restart_file), ESMF_LOGMSG_INFO) From e37038de42f7820295cfea9a3651db4323a9f4f7 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 12 Mar 2021 16:19:32 -0700 Subject: [PATCH 24/42] correct grid for nuopc --- cime_config/testdefs/testlist_drv.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cime_config/testdefs/testlist_drv.xml b/cime_config/testdefs/testlist_drv.xml index fcd1cd61c..80b28a301 100644 --- a/cime_config/testdefs/testlist_drv.xml +++ b/cime_config/testdefs/testlist_drv.xml @@ -155,7 +155,7 @@ - + @@ -394,18 +394,18 @@ - - - + + + - - - + + + From cd0e92270b04f94c5bf2f748f96a173f64b1cf0b Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Sun, 14 Mar 2021 16:05:58 -0600 Subject: [PATCH 25/42] added attribute for scol_spval for single column checks to be consistent across all component caps --- cime_config/config_component.xml | 100 ------------------------------- drivers/cime/esm.F90 | 23 ++++--- 2 files changed, 15 insertions(+), 108 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index e14af5aab..f48a1155a 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1267,14 +1267,6 @@ - - - - - - - - char UNSET @@ -1282,22 +1274,6 @@ env_run.xml atm mesh file (full pathname) - - - char - UNSET - run_domain - env_run.xml - lnd domain file - - - char - $DIN_LOC_ROOT/share/domains - run_domain - env_run.xml - path of lnd domain file - - char UNSET @@ -1305,21 +1281,6 @@ env_run.xml lnd mesh file (full pathname) - - - - - - - - - - - - - - - char UNSET @@ -1327,21 +1288,6 @@ env_run.xml rof mesh file (full pathname) - - - - - - - - - - - - - - - char UNSET @@ -1349,21 +1295,6 @@ env_run.xml wav mesh file (full pathname) - - - - - - - - - - - - - - - char UNSET @@ -1371,21 +1302,6 @@ env_run.xml ice mesh file (full pahtname) - - - - - - - - - - - - - - - char UNSET @@ -1393,21 +1309,6 @@ env_run.xml ocn mesh file (full pathname) - - - - - - - - - - - - - - - char UNSET @@ -1415,7 +1316,6 @@ env_run.xml glc mesh file (full pathname) - char UNSET diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index 09e0c1eed..8d62bdbd3 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -1137,12 +1137,18 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) call shr_sys_abort(subname//' ERROR: '//trim(compname)//' both scol_lon and scol_lat must be greater than -999 ') end if - if (scol_lon > -999. .and. scol_lat > -999.) then + ! Set the special value for single column - if pts_lat or pts_lon are equal to the special value + ! in the component cap - then single column is not activated + write(cvalue,*) scol_spval + call NUOPC_CompAttributeSet(gcomp, name='scol_spval', value=trim(cvalue), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + if (scol_lon > scol_spval .and. scol_lat > scol_spval) then ! NOTE: currently assume that single column capability is restricted to - ! ATM, OCN and ICE components only - ! verify that ROF, WAV and LND are not trying to use single column mode - if (trim(compname) == 'ROF' .or. trim(compname) == 'WAV') then + ! ATM, LND, OCN and ICE components only + ! verify that WAV and LND are not trying to use single column mode + if (trim(compname) == 'WAV' .or. trim(compname) == 'ROF' .or. trim(compname) == 'GLC') then call shr_sys_abort(subname//' ERROR: '//trim(compname)//' does not support single column mode ') end if @@ -1155,8 +1161,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) call shr_sys_abort(subname//' ERROR: single column mode must be run on 1 pe') endif - write(logunit,'(a,2(f10.5,2x))')trim(subname)//' single column point for '//trim(compname)//' has lon and lat = ',& - scol_lon,scol_lat + write(logunit,'(a,2(f10.5,2x))')trim(subname)//' single column point for '//trim(compname)//& + ' has lon and lat = ',scol_lon,scol_lat ! This is either a single column or a single point so add attributes call NUOPC_CompAttributeAdd(gcomp, & @@ -1166,10 +1172,11 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) 'scol_lndmask', & 'scol_lndfrac', & 'scol_ocnmask', & - 'scol_ocnfrac'/), rc=rc) + 'scol_ocnfrac', & + 'scol_spval '/), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - if (trim(single_column_lnd_domainfile) /= 'null') then + if (trim(single_column_lnd_domainfile) /= 'UNSET') then ! In this case the domain file is not a single point file - but normally a ! global domain file where a nearest neighbor search will be done to find From 6c138135bc89bdfc013d04bf5bdf7843d3ad1aa0 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 16 Mar 2021 08:57:43 -0600 Subject: [PATCH 26/42] update esmf version in github action --- .github/workflows/extbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/extbuild.yml b/.github/workflows/extbuild.yml index 3bedfc157..fea7638ae 100644 --- a/.github/workflows/extbuild.yml +++ b/.github/workflows/extbuild.yml @@ -19,7 +19,7 @@ jobs: CXX: mpicxx CPPFLAGS: "-I/usr/include -I/usr/local/include" # Versions of all dependencies can be updated here - ESMF_VERSION: ESMF_8_1_0_beta_snapshot_41 + ESMF_VERSION: ESMF_8_1_0_beta_snapshot_47 PNETCDF_VERSION: pnetcdf-1.12.2 NETCDF_FORTRAN_VERSION: v4.5.2 # PIO version is awkward From d8c48c6fe51966b6f22440a46ec88a8210f4d8f7 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 16 Mar 2021 09:40:54 -0600 Subject: [PATCH 27/42] update pio version --- .github/workflows/extbuild.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extbuild.yml b/.github/workflows/extbuild.yml index fea7638ae..f3563cbad 100644 --- a/.github/workflows/extbuild.yml +++ b/.github/workflows/extbuild.yml @@ -23,8 +23,8 @@ jobs: PNETCDF_VERSION: pnetcdf-1.12.2 NETCDF_FORTRAN_VERSION: v4.5.2 # PIO version is awkward - PIO_VERSION_DIR: pio2_5_2 - PIO_VERSION: pio-2.5.2 + PIO_VERSION_DIR: pio2_5_3 + PIO_VERSION: pio-2.5.3 steps: - uses: actions/checkout@v2 # Build the ESMF library, if the cache contains a previous build From bf16a6b6877dd0634b55b23ee637aabe84690d3f Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 16 Mar 2021 10:03:40 -0600 Subject: [PATCH 28/42] fixed comments in PR --- drivers/cime/esm.F90 | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index 7c30e0efd..e517e8f9e 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -1078,7 +1078,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) ! Generate a mesh for single column - use netcdf + use netcdf, only : nf90_open, nf90_close, nf90_noerr + use netcdf, only : nf90_inq_dimid, inf90_inquire_dimension, nf90_inq_varid, nf90_getvar use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet, ESMF_SUCCESS @@ -1185,22 +1186,16 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) ! the closest point in the domin file to scol_lon and scol_lat status = nf90_open(single_column_lnd_domainfile, NF90_NOWRITE, ncid) - if (status /= nf90_noerr) then - call shr_sys_abort (trim(subname) //': Cannot open '//trim(single_column_lnd_domainfile)) - endif - status = nf90_inq_dimid (ncid, 'ni', dimid) - status = nf90_inquire_dimension(ncid, dimid, len=ni) - status = nf90_inq_dimid (ncid, 'nj', dimid) - status = nf90_inquire_dimension(ncid, dimid, len=nj) - - status = nf90_open(single_column_lnd_domainfile, NF90_NOWRITE, ncid) - if (status /= nf90_noerr) then - call shr_sys_abort (trim(subname) //': Cannot open '//trim(single_column_lnd_domainfile)) - endif + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': opening '//& + trim(single_column_lnd_domainfile)) status = nf90_inq_dimid (ncid, 'ni', dimid) + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': inq_dimid ni') status = nf90_inquire_dimension(ncid, dimid, len=ni) + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': inquire_dimension ni') status = nf90_inq_dimid (ncid, 'nj', dimid) + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': inq_dimid nj') status = nf90_inquire_dimension(ncid, dimid, len=nj) + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': inquire_dimension nj') status = nf90_inq_varid(ncid, 'xc' , varid_xc) if (status /= nf90_noerr) call shr_sys_abort (subname//' inq_varid xc') @@ -1288,6 +1283,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return status = nf90_close(ncid) + if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': closing '//& + trim(single_column_lnd_domainfile)) write(logunit,'(a,2(f13.5,2x))')trim(subname)//' nearest neighbor scol_lon and scol_lat in '& //trim(single_column_lnd_domainfile)//' are ',scol_lon,scol_lat From 839e2940152af9d377aa42d5bc8d0cdd088696e9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 16 Mar 2021 10:14:38 -0600 Subject: [PATCH 29/42] add bumpversion action --- .github/workflows/bumpversion.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/bumpversion.yml diff --git a/.github/workflows/bumpversion.yml b/.github/workflows/bumpversion.yml new file mode 100644 index 000000000..d32f4a818 --- /dev/null +++ b/.github/workflows/bumpversion.yml @@ -0,0 +1,20 @@ +name: Bump version +on: + push: + branches: + - master +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' + - name: Bump version and push tag + uses: anothrNick/github-tag-action@1.26.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: minor + RELEASE_BRANCHES: master + DRY_RUN: True From b4d2b192174f343dfcf8d4f59b09cb75532e9239 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Tue, 16 Mar 2021 20:15:00 -0600 Subject: [PATCH 30/42] compilation bug fixes --- drivers/cime/esm.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index e517e8f9e..44dc74a51 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -1078,8 +1078,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc) ! Generate a mesh for single column - use netcdf, only : nf90_open, nf90_close, nf90_noerr - use netcdf, only : nf90_inq_dimid, inf90_inquire_dimension, nf90_inq_varid, nf90_getvar + use netcdf, only : nf90_open, nf90_close, nf90_noerr, nf90_nowrite + use netcdf, only : nf90_inq_dimid, nf90_inquire_dimension, nf90_inq_varid, nf90_get_var use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet, ESMF_SUCCESS From ad849ecbe9d46e29008dfbe7ca0cdc57739bbe22 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 17 Mar 2021 07:55:03 -0600 Subject: [PATCH 31/42] pr to impliment automatic versioning in cmeps repo --- .github/workflows/bumpversion.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bumpversion.yml b/.github/workflows/bumpversion.yml index d32f4a818..c682973c4 100644 --- a/.github/workflows/bumpversion.yml +++ b/.github/workflows/bumpversion.yml @@ -17,4 +17,4 @@ jobs: WITH_V: true DEFAULT_BUMP: minor RELEASE_BRANCHES: master - DRY_RUN: True + DRY_RUN: False From 014c22363a153382812e3a3feccbf7a35cad1493 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 19 Mar 2021 09:38:15 -0600 Subject: [PATCH 32/42] changes to get ctsm LILAC mode to work --- cime_config/config_component.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index f48a1155a..34e2d1965 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1267,6 +1267,23 @@ + + + + char + UNSET + run_domain + env_run.xml + lnd domain file + + + char + $DIN_LOC_ROOT/share/domains + run_domain + env_run.xml + path of lnd domain file + + char UNSET From 92df294b237cac3ce8daac9ab76902890394b5d6 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Sun, 21 Mar 2021 14:07:08 -0600 Subject: [PATCH 33/42] addition of bilnr_cart3d mapping --- mediator/esmFlds.F90 | 14 ++++++++------ mediator/med_map_mod.F90 | 41 +++++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/mediator/esmFlds.F90 b/mediator/esmFlds.F90 index 5e0430b57..4f4d938c9 100644 --- a/mediator/esmFlds.F90 +++ b/mediator/esmFlds.F90 @@ -48,12 +48,13 @@ module esmflds integer , public, parameter :: mapnstod_consd = 7 ! nearest source to destination followed by conservative dst integer , public, parameter :: mapnstod_consf = 8 ! nearest source to destination followed by conservative frac integer , public, parameter :: mappatch_uv3d = 9 ! rotate u,v to 3d cartesian space, map from src->dest, then rotate back - integer , public, parameter :: map_rof2ocn_ice = 10 ! custom smoothing map to map ice from rof->ocn (cesm only) - integer , public, parameter :: map_rof2ocn_liq = 11 ! custom smoothing map to map liq from rof->ocn (cesm only) - integer , public, parameter :: map_glc2ocn_liq = 12 ! custom smoothing map to map liq from glc->ocn (cesm only) - integer , public, parameter :: map_glc2ocn_ice = 13 ! custom smoothing map to map ice from glc->ocn (cesm only) - integer , public, parameter :: mapfillv_bilnr = 14 ! fill value followed by bilinear - integer , public, parameter :: nmappers = 14 + integer , public, parameter :: mapbilnr_uv3d = 10 ! rotate u,v to 3d cartesian space, map from src->dest, then rotate back + integer , public, parameter :: map_rof2ocn_ice = 11 ! custom smoothing map to map ice from rof->ocn (cesm only) + integer , public, parameter :: map_rof2ocn_liq = 12 ! custom smoothing map to map liq from rof->ocn (cesm only) + integer , public, parameter :: map_glc2ocn_liq = 13 ! custom smoothing map to map liq from glc->ocn (cesm only) + integer , public, parameter :: map_glc2ocn_ice = 14 ! custom smoothing map to map ice from glc->ocn (cesm only) + integer , public, parameter :: mapfillv_bilnr = 15 ! fill value followed by bilinear + integer , public, parameter :: nmappers = 15 character(len=*) , public, parameter :: mapnames(nmappers) = & (/'bilnr ',& @@ -65,6 +66,7 @@ module esmflds 'nstod_consd',& 'nstod_consf',& 'patch_uv3d ',& + 'bilnr_uv3d ',& 'rof2ocn_ice',& 'rof2ocn_liq',& 'glc2ocn_ice',& diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 24352bdca..e8ff18991 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -211,7 +211,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, use ESMF , only : ESMF_REGRIDMETHOD_BILINEAR, ESMF_REGRIDMETHOD_PATCH use ESMF , only : ESMF_REGRIDMETHOD_CONSERVE, ESMF_NORMTYPE_DSTAREA, ESMF_NORMTYPE_FRACAREA use ESMF , only : ESMF_UNMAPPEDACTION_IGNORE, ESMF_REGRIDMETHOD_NEAREST_STOD - use esmFlds , only : mapbilnr, mapconsf, mapconsd, mappatch, mappatch_uv3d, mapfcopy + use esmFlds , only : mapbilnr, mapconsf, mapconsd, mappatch, mappatch_uv3d, mapbilnr_uv3d, mapfcopy use esmFlds , only : mapunset, mapnames, nmappers use esmFlds , only : mapnstod, mapnstod_consd, mapnstod_consf, mapnstod_consd use esmFlds , only : mapfillv_bilnr @@ -305,19 +305,21 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, ignoreUnmatchedIndices=.true., & srcTermProcessing=srcTermProcessing_Value, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - else if (mapindex == mapbilnr) then - if (mastertask) then - write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) + else if (mapindex == mapbilnr .or. mapindex == mapbilnr_uv3d) then + if (.not. ESMF_RouteHandleIsCreated(routehandles(mapbilnr))) then + if (mastertask) then + write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) + end if + call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mapbilnr), & + srcMaskValues=(/srcMaskValue/), & + dstMaskValues=(/dstMaskValue/), & + regridmethod=ESMF_REGRIDMETHOD_BILINEAR, & + polemethod=polemethod, & + srcTermProcessing=srcTermProcessing_Value, & + ignoreDegenerate=.true., & + unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return end if - call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mapbilnr), & - srcMaskValues=(/srcMaskValue/), & - dstMaskValues=(/dstMaskValue/), & - regridmethod=ESMF_REGRIDMETHOD_BILINEAR, & - polemethod=polemethod, & - srcTermProcessing=srcTermProcessing_Value, & - ignoreDegenerate=.true., & - unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return else if (mapindex == mapfillv_bilnr) then if (mastertask) then write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) @@ -566,7 +568,7 @@ subroutine med_map_mapnorm_init(gcomp, rc) call ESMF_FieldGet(fieldlist(1), mesh=mesh_dst, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Createis_local%wrap%field_NormOne(n1,n2,m) + ! Create is_local%wrap%field_NormOne(n1,n2,m) do m = 1,nmappers if (med_map_RH_is_created(is_local%wrap%RH,n1,n2,m,rc=rc)) then is_local%wrap%field_NormOne(n1,n2,m) = ESMF_FieldCreate(mesh_dst, & @@ -791,7 +793,8 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d use ESMF , only : ESMF_FieldBundle, ESMF_FieldBundleGet use ESMF , only : ESMF_FieldBundleIsCreated use ESMF , only : ESMF_FieldRedist, ESMF_RouteHandle - use esmFlds , only : nmappers, mapfcopy, mappatch_uv3d, mappatch + use esmFlds , only : nmappers, mapfcopy + use esmFlds , only : mappatch_uv3d, mappatch, mapbilnr_uv3d, mapbilnr use med_internalstate_mod , only : packed_data_type ! input/output variables @@ -853,6 +856,12 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d call med_map_uv_cart3d(FBsrc, FBdst, routehandles, mappatch, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + else if (mapindex == mapbilnr_uv3d) then + + ! For mapbilnr_uv3d do not use packed field bundles + call med_map_uv_cart3d(FBsrc, FBdst, routehandles, mapbilnr, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + else ! ----------------------------------- @@ -919,6 +928,8 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d else if ( trim(packed_data(mapindex)%mapnorm) == 'one' .or. trim(packed_data(mapindex)%mapnorm) == 'none') then + write(6,*)'DEBUG: maptype = ',mapindex + ! Mapping with no normalization that is not redistribution call med_map_field (& field_src=packed_data(mapindex)%field_src, & From 04b8b08a00c38fdea758360283799b2c6cb6b953 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Mon, 22 Mar 2021 11:08:24 -0600 Subject: [PATCH 34/42] removed extraneous comment --- mediator/med_map_mod.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index e8ff18991..01892601d 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -928,8 +928,6 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d else if ( trim(packed_data(mapindex)%mapnorm) == 'one' .or. trim(packed_data(mapindex)%mapnorm) == 'none') then - write(6,*)'DEBUG: maptype = ',mapindex - ! Mapping with no normalization that is not redistribution call med_map_field (& field_src=packed_data(mapindex)%field_src, & From aa866266de78642fd480765f02b935dfb4b779c0 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Mon, 22 Mar 2021 14:26:03 -0400 Subject: [PATCH 35/42] update datm dependence of srcMaskValue --- mediator/med_map_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 3b9142089..ae9e63dbd 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -259,7 +259,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, if (n1 == compatm .and. (n2 == compocn .or. n2 == compice)) then srcMaskValue = 1 dstMaskValue = 0 - if (atm_name(1:9) == 'datm_cfsr' .or. atm_name(1:9) == 'datm_gefs' ) then + if (trim(atm_name) == 'datm') then srcMaskValue = 0 endif else if (n2 == compatm .and. (n1 == compocn .or. n1 == compice)) then From f1e95074fbf75095d4edbb33be6a245639a6fcf4 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 24 Mar 2021 08:35:14 -0400 Subject: [PATCH 36/42] clean branch for dststatus and mapbilnr_nstod *add dststatus field retrieval and optional write to file *add mapbilnr_nstop mapping type *add abort of coupling_mode is not supported *trailing whitespace cleanup --- mediator/esmFlds.F90 | 6 +- mediator/esmFldsExchange_hafs_mod.F90 | 2 +- mediator/fd_nems.yaml | 52 ++++++++--------- mediator/med.F90 | 7 ++- mediator/med_diag_mod.F90 | 14 ++--- mediator/med_fraction_mod.F90 | 4 +- mediator/med_io_mod.F90 | 16 +++--- mediator/med_map_mod.F90 | 83 ++++++++++++++++++++++++++- mediator/med_phases_post_ice_mod.F90 | 2 +- mediator/med_phases_prep_glc_mod.F90 | 22 +++---- mediator/med_phases_prep_ice_mod.F90 | 4 +- mediator/med_phases_prep_lnd_mod.F90 | 2 +- 12 files changed, 149 insertions(+), 65 deletions(-) diff --git a/mediator/esmFlds.F90 b/mediator/esmFlds.F90 index 4f4d938c9..9a937ebfa 100644 --- a/mediator/esmFlds.F90 +++ b/mediator/esmFlds.F90 @@ -54,7 +54,8 @@ module esmflds integer , public, parameter :: map_glc2ocn_liq = 13 ! custom smoothing map to map liq from glc->ocn (cesm only) integer , public, parameter :: map_glc2ocn_ice = 14 ! custom smoothing map to map ice from glc->ocn (cesm only) integer , public, parameter :: mapfillv_bilnr = 15 ! fill value followed by bilinear - integer , public, parameter :: nmappers = 15 + integer , public, parameter :: mapbilnr_nstod = 16 ! bilinear with nstod extrapolation + integer , public, parameter :: nmappers = 16 character(len=*) , public, parameter :: mapnames(nmappers) = & (/'bilnr ',& @@ -71,7 +72,8 @@ module esmflds 'rof2ocn_liq',& 'glc2ocn_ice',& 'glc2ocn_liq',& - 'fillv_bilnr'/) + 'fillv_bilnr',& + 'bilnr_nstod'/) !----------------------------------------------- ! Set coupling mode diff --git a/mediator/esmFldsExchange_hafs_mod.F90 b/mediator/esmFldsExchange_hafs_mod.F90 index 37e8cb3e3..1786f3684 100644 --- a/mediator/esmFldsExchange_hafs_mod.F90 +++ b/mediator/esmFldsExchange_hafs_mod.F90 @@ -187,7 +187,7 @@ subroutine esmFldsExchange_hafs_advt(gcomp, phase, rc) call addfld(fldListTo(compatm)%flds, 'So_ofrac') !---------------------------------------------------------- - ! to atm: surface temperatures from ocn + ! to atm: surface temperatures from ocn !---------------------------------------------------------- call addfld(fldListFr(compocn)%flds, 'So_t') call addfld(fldListTo(compatm)%flds, 'So_t') diff --git a/mediator/fd_nems.yaml b/mediator/fd_nems.yaml index 1fc70726d..75db4177d 100644 --- a/mediator/fd_nems.yaml +++ b/mediator/fd_nems.yaml @@ -575,7 +575,7 @@ description: atmosphere export - latent heat flux conversion - alias: Faox_evap standard_name : mean_evap_rate - description: mediator calculation - atm/ocn specific humidity flux + description: mediator calculation - atm/ocn specific humidity flux # #----------------------------------- # section: atmosphere fields that need to be defined but are not used @@ -694,8 +694,8 @@ #----------------------------------- # - standard_name: sea_surface_height_above_sea_level - canonical_units: m - description: ww3 import + canonical_units: m + description: ww3 import # - standard_name: sea_surface_salinity alias: s_surf @@ -705,22 +705,22 @@ - standard_name: surface_eastward_sea_water_velocity alias: ocn_current_zonal canonical_units: m s-1 - description: ww3 import + description: ww3 import # - standard_name: surface_northward_sea_water_velocity alias: ocn_current_merid canonical_units: m s-1 - description: ww3 import + description: ww3 import # - standard_name: eastward_wind_at_10m_height alias: inst_zonal_wind_height10m canonical_units: m s-1 - description: ww3 import + description: ww3 import # - standard_name: northward_wind_at_10m_height alias: inst_merid_wind_height10m canonical_units: m s-1 - description: ww3 import + description: ww3 import # - standard_name: sea_ice_concentration alias: ice_fraction @@ -732,77 +732,77 @@ # - standard_name: wave_induced_charnock_parameter canonical_units: 1 - description: ww3 export + description: ww3 export # - standard_name: wave_z0_roughness_length canonical_units: 1 - description: ww3 export + description: ww3 export # - standard_name: northward_stokes_drift_current canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_stokes_drift_current canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_partitioned_stokes_drift_1 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_partitioned_stokes_drift_2 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_partitioned_stokes_drift_3 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: northward_partitioned_stokes_drift_1 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: northward_partitioned_stokes_drift_2 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: northward_partitioned_stokes_drift_3 canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_wave_bottom_current canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: northward_wave_bottom_current canonical_units: m s-1 - description: ww3 export + description: ww3 export # - standard_name: wave_bottom_current_radian_frequency canonical_units: rad s-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_wave_radiation_stress_gradient canonical_units: Pa - description: ww3 export + description: ww3 export # - standard_name: northward_wave_radiation_stress_gradient canonical_units: Pa - description: ww3 export + description: ww3 export # - standard_name: eastward_wave_radiation_stress canonical_units: N m-1 - description: ww3 export + description: ww3 export # - standard_name: eastward_northward_wave_radiation_stress canonical_units: N m-1 - description: ww3 export + description: ww3 export # - standard_name: wave_bottom_current_period canonical_units: s - description: ww3 export + description: ww3 export # - standard_name: northward_wave_radiation_stress canonical_units: Pa - description: ww3 export + description: ww3 export # diff --git a/mediator/med.F90 b/mediator/med.F90 index ecc17851d..06a2e61af 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -644,6 +644,7 @@ subroutine InitializeIPDv03p1(gcomp, importState, exportState, clock, rc) use ESMF , only : ESMF_GridComp, ESMF_State, ESMF_Clock, ESMF_SUCCESS, ESMF_LogFoundAllocError use ESMF , only : ESMF_LogMsg_Info, ESMF_LogWrite + use ESMF , only : ESMF_END_ABORT, ESMF_Finalize use NUOPC , only : NUOPC_AddNamespace, NUOPC_Advertise, NUOPC_AddNestedState use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd @@ -749,12 +750,16 @@ subroutine InitializeIPDv03p1(gcomp, importState, exportState, clock, rc) if (trim(coupling_mode) == 'cesm') then call esmFldsExchange_cesm(gcomp, phase='advertise', rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - else if (trim(coupling_mode(1:4)) == 'nems') then + else if (trim(coupling_mode) == 'nems_orig' .or. trim(coupling_mode) == 'nems_frac' & + .or. trim(coupling_mode) == 'nems_orig_data') then call esmFldsExchange_nems(gcomp, phase='advertise', rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return else if (trim(coupling_mode(1:4)) == 'hafs') then call esmFldsExchange_hafs(gcomp, phase='advertise', rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + else + call ESMF_LogWrite(trim(coupling_mode)//' is not a valid coupling_mode', ESMF_LOGMSG_INFO) + call ESMF_Finalize(endflag=ESMF_END_ABORT) end if !------------------ diff --git a/mediator/med_diag_mod.F90 b/mediator/med_diag_mod.F90 index c191b7121..72295a5ac 100644 --- a/mediator/med_diag_mod.F90 +++ b/mediator/med_diag_mod.F90 @@ -22,7 +22,7 @@ module med_diag_mod use ESMF , only : ESMF_GridComp, ESMF_Clock, ESMF_Time use ESMF , only : ESMF_VM, ESMF_VMReduce, ESMF_REDUCE_SUM use ESMF , only : ESMF_GridCompGet, ESMF_ClockGet, ESMF_TimeGet - use ESMF , only : ESMF_Alarm, ESMF_ClockGetAlarm, ESMF_AlarmIsRinging, ESMF_AlarmRingerOff + use ESMF , only : ESMF_Alarm, ESMF_ClockGetAlarm, ESMF_AlarmIsRinging, ESMF_AlarmRingerOff use ESMF , only : ESMF_FieldBundle, ESMF_Field, ESMF_FieldGet use shr_const_mod , only : shr_const_rearth, shr_const_pi, shr_const_latice use shr_const_mod , only : shr_const_ice_ref_sal, shr_const_ocn_ref_sal, shr_const_isspval @@ -800,7 +800,7 @@ subroutine diag_atm_wiso_recv(FB, fldname, nf_16O, nf_18O, nf_HDO, areas, lats, integer , intent(out) :: rc ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:,:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS @@ -857,7 +857,7 @@ subroutine diag_atm_wiso_send(FB, fldname, nf_16O, nf_18O, nf_HDO, areas, lats, integer , intent(out) :: rc ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:,:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS @@ -1575,7 +1575,7 @@ subroutine diag_ice_recv(FB, fldname, nf, areas, lats, ifrac, budget, minus, sca integer , intent(out) :: rc ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS @@ -1621,7 +1621,7 @@ subroutine diag_ice_recv_wiso(FB, fldname, nf_16O, nf_18O, nf_HDO, areas, lats, integer , intent(out) :: rc ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:,:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS @@ -1754,7 +1754,7 @@ subroutine diag_ice_send(FB, fldname, nf, areas, lats, ifrac, budget, rc) integer , intent(out) :: rc ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS @@ -1788,7 +1788,7 @@ subroutine diag_ice_send_wiso(FB, fldname, nf_16O, nf_18O, nf_HDO, areas, lats, ! local variables integer :: n, ip - type(ESMF_Field) :: lfield + type(ESMF_Field) :: lfield real(r8), pointer :: data(:,:) => null() ! ------------------------------------------------------------------ rc = ESMF_SUCCESS diff --git a/mediator/med_fraction_mod.F90 b/mediator/med_fraction_mod.F90 index 0d083f0e6..018b4339c 100644 --- a/mediator/med_fraction_mod.F90 +++ b/mediator/med_fraction_mod.F90 @@ -295,7 +295,7 @@ subroutine med_fraction_init(gcomp, rc) end if !--------------------------------------- - ! Set 'ofrac' in FBFrac(compocn) + ! Set 'ofrac' in FBFrac(compocn) !--------------------------------------- if (is_local%wrap%comp_present(compocn)) then @@ -681,7 +681,7 @@ subroutine med_fraction_set(gcomp, rc) ! The model mask is normally assumed to be an selected ocean mask from a fully coupled run ! So in it is (1-land fraction) on the atm grid - ! set ifrac + ! set ifrac if (associated(ifrac)) then ifrac(:) = Si_ifrac(:) * Si_imask(:) endif diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90 index a13d66762..d4f767d6e 100644 --- a/mediator/med_io_mod.F90 +++ b/mediator/med_io_mod.F90 @@ -181,7 +181,7 @@ subroutine med_io_init(gcomp, rc) call ESMF_VMGet(vm, mpiCommunicator=comm, localPet=localPet, petCount=petCount, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! query component specific PIO attributes + ! query component specific PIO attributes ! pio_netcdf_format call NUOPC_CompAttributeGet(gcomp, name='pio_netcdf_format', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -212,9 +212,9 @@ subroutine med_io_init(gcomp, rc) if (isPresent .and. isSet) then cvalue = ESMF_UtilStringUpperCase(cvalue) if (trim(cvalue) .eq. 'NETCDF') then - pio_iotype = PIO_IOTYPE_NETCDF + pio_iotype = PIO_IOTYPE_NETCDF else if (trim(cvalue) .eq. 'PNETCDF') then - pio_iotype = PIO_IOTYPE_PNETCDF + pio_iotype = PIO_IOTYPE_PNETCDF else if (trim(cvalue) .eq. 'NETCDF4C') then pio_iotype = PIO_IOTYPE_NETCDF4C else if (trim(cvalue) .eq. 'NETCDF4P') then @@ -276,7 +276,7 @@ subroutine med_io_init(gcomp, rc) write(logunit,*) ' parallel io requires at least two io pes - following parameters are updated:' write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks - end if + end if endif ! check/set/correct io pio parameters @@ -294,7 +294,7 @@ subroutine med_io_init(gcomp, rc) end if if (pio_stride == 1) then pio_root = 0 - endif + endif if (pio_root + (pio_stride)*(pio_numiotasks-1) >= petCount .or. & pio_stride <= 0 .or. pio_numiotasks <= 0 .or. pio_root < 0 .or. pio_root > petCount-1) then @@ -366,8 +366,8 @@ subroutine med_io_init(gcomp, rc) ! set PIO debug level call pio_setdebuglevel(pio_debug_level) - ! query shared PIO rearranger attributes - ! pio_rearr_comm_type + ! query shared PIO rearranger attributes + ! pio_rearr_comm_type call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_type', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -386,7 +386,7 @@ subroutine med_io_init(gcomp, rc) cvalue = 'P2P' pio_rearr_comm_type = PIO_REARR_COMM_P2P end if - if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_type = ', trim(cvalue), pio_rearr_comm_type + if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_type = ', trim(cvalue), pio_rearr_comm_type ! pio_rearr_comm_fcd call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_fcd', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 01892601d..b6fbd2220 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -1,6 +1,7 @@ module med_map_mod use med_kind_mod , only : CX=>SHR_KIND_CX, CS=>SHR_KIND_CS, CL=>SHR_KIND_CL, R8=>SHR_KIND_R8 + use med_kind_mod , only : I4=>SHR_KIND_I4 use ESMF , only : ESMF_SUCCESS, ESMF_FAILURE use ESMF , only : ESMF_LOGMSG_ERROR, ESMF_LOGMSG_INFO, ESMF_LogWrite use ESMF , only : ESMF_Field @@ -207,17 +208,21 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, use ESMF , only : ESMF_RouteHandle, ESMF_RouteHandlePrint, ESMF_Field, ESMF_MAXSTR use ESMF , only : ESMF_PoleMethod_Flag, ESMF_POLEMETHOD_ALLAVG use ESMF , only : ESMF_FieldSMMStore, ESMF_FieldRedistStore, ESMF_FieldRegridStore - use ESMF , only : ESMF_RouteHandleIsCreated + use ESMF , only : ESMF_RouteHandleIsCreated use ESMF , only : ESMF_REGRIDMETHOD_BILINEAR, ESMF_REGRIDMETHOD_PATCH use ESMF , only : ESMF_REGRIDMETHOD_CONSERVE, ESMF_NORMTYPE_DSTAREA, ESMF_NORMTYPE_FRACAREA use ESMF , only : ESMF_UNMAPPEDACTION_IGNORE, ESMF_REGRIDMETHOD_NEAREST_STOD + use ESMF , only : ESMF_EXTRAPMETHOD_NEAREST_STOD + use ESMF , only : ESMF_Mesh, ESMF_MeshLoc, ESMF_MESHLOC_ELEMENT, ESMF_TYPEKIND_I4 + use ESMF , only : ESMF_MeshGet, ESMF_DistGridGet, ESMF_DistGrid, ESMF_TYPEKIND_R8 + use ESMF , only : ESMF_FieldGet, ESMF_FieldCreate, ESMF_FieldWrite, ESMF_FieldDestroy use esmFlds , only : mapbilnr, mapconsf, mapconsd, mappatch, mappatch_uv3d, mapbilnr_uv3d, mapfcopy use esmFlds , only : mapunset, mapnames, nmappers use esmFlds , only : mapnstod, mapnstod_consd, mapnstod_consf, mapnstod_consd - use esmFlds , only : mapfillv_bilnr + use esmFlds , only : mapfillv_bilnr, mapbilnr_nstod use esmFlds , only : ncomps, compatm, compice, compocn, compname use esmFlds , only : mapfcopy, mapconsd, mapconsf, mapnstod - use esmFlds , only : coupling_mode, compname + use esmFlds , only : coupling_mode use esmFlds , only : atm_name use med_constants_mod , only : ispval_mask => med_constants_ispval_mask @@ -232,12 +237,19 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, integer , intent(out) :: rc ! local variables + type(ESMF_Mesh) :: dstmesh + type(ESMF_Field) :: dststatusfield, doffield + type(ESMF_DistGrid) :: distgrid character(len=CS) :: string character(len=CS) :: mapname + character(len=CL) :: fname integer :: srcMaskValue integer :: dstMaskValue character(len=ESMF_MAXSTR) :: lmapfile logical :: rhprint = .false. + logical :: dststatus_print = .false. + integer :: ns + integer(I4), pointer :: dof(:) => null() integer :: srcTermProcessing_Value = 0 type(ESMF_PoleMethod_Flag), parameter :: polemethod=ESMF_POLEMETHOD_ALLAVG character(len=*), parameter :: subname=' (module_med_map: med_map_routehandles_initfrom_field) ' @@ -249,6 +261,13 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, end if mapname = trim(mapnames(mapindex)) + call ESMF_LogWrite(trim(subname)//": mapname "//trim(mapname), ESMF_LOGMSG_INFO) + + ! create a field to retrieve the dststatus field + call ESMF_FieldGet(flddst, mesh=dstmesh, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + dststatusfield = ESMF_FieldCreate(dstmesh, ESMF_TYPEKIND_I4, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return if (trim(coupling_mode) == 'cesm') then dstMaskValue = ispval_mask @@ -317,6 +336,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, polemethod=polemethod, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return end if @@ -331,6 +351,22 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, polemethod=polemethod, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & + unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + else if (mapindex == mapbilnr_nstod) then + if (mastertask) then + write(logunit,'(A)') trim(subname)//' creating RH '//trim(mapname)//' for '//trim(string) + end if + call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mapbilnr_nstod), & + srcMaskValues=(/srcMaskValue/), & + dstMaskValues=(/dstMaskValue/), & + regridmethod=ESMF_REGRIDMETHOD_BILINEAR, & + extrapMethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & + polemethod=polemethod, & + srcTermProcessing=srcTermProcessing_Value, & + ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return else if (mapindex == mapconsf .or. mapindex == mapnstod_consf) then @@ -344,6 +380,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, normType=ESMF_NORMTYPE_FRACAREA, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return @@ -358,6 +395,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, normType=ESMF_NORMTYPE_DSTAREA, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return @@ -386,6 +424,33 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, return end if + ! Output destination status field to file if requested + if (dststatus_print) then + if (mapindex /= mapfcopy .or. lmapfile /= 'unset') then + fname = 'dststatus.'//trim(compname(n1))//'.'//trim(compname(n2))//'.'//trim(mapname)//'.nc' + call ESMF_LogWrite(trim(subname)//": writing dstStatusField to "//trim(fname), ESMF_LOGMSG_INFO) + + call ESMF_FieldWrite(dststatusfield, filename=trim(fname), variableName='dststatus', & + overwrite=.true., rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! the sequence index in order to sort the dststatus field + call ESMF_MeshGet(dstmesh, elementDistgrid=distgrid, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_DistGridGet(distgrid, localDE=0, elementCount=ns, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + allocate(dof(ns)) + call ESMF_DistGridGet(distgrid, localDE=0, seqIndexList=dof, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + doffield = ESMF_FieldCreate(dstmesh, dof, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldWrite(doffield, fileName='dof.'//trim(compname(n2))//'.nc', variableName='dof', & + overwrite=.true., rc=rc) + deallocate(dof) + call ESMF_FieldDestroy(doffield, rc=rc, noGarbage=.true.) + end if + end if + ! consd_nstod method requires a second routehandle if (mapindex == mapnstod .or. mapindex == mapnstod_consd .or. mapindex == mapnstod_consf) then call ESMF_FieldRegridStore(fldsrc, flddst, routehandle=routehandles(mapnstod), & @@ -394,9 +459,19 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, regridmethod=ESMF_REGRIDMETHOD_NEAREST_STOD, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! Output destination status field to file if requested + if (dststatus_print) then + fname = 'dststatus.'//trim(compname(n1))//'.'//trim(compname(n2))//'.'//trim(mapname)//'_2.nc' + call ESMF_LogWrite(trim(subname)//": writing dstStatusField to "//trim(fname), ESMF_LOGMSG_INFO) + + call ESMF_FieldWrite(dststatusfield, filename=trim(fname), variableName='dststatus', overwrite=.true., rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + end if end if ! Check that a valid route handle has been created @@ -416,6 +491,8 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, if (chkerr(rc,__LINE__,u_FILE_u)) return endif + call ESMF_FieldDestroy(dststatusfield, rc=rc, noGarbage=.true.) + end subroutine med_map_routehandles_initfrom_field !================================================================================ diff --git a/mediator/med_phases_post_ice_mod.F90 b/mediator/med_phases_post_ice_mod.F90 index b7bfbb679..f605006e5 100644 --- a/mediator/med_phases_post_ice_mod.F90 +++ b/mediator/med_phases_post_ice_mod.F90 @@ -25,7 +25,7 @@ subroutine med_phases_post_ice(gcomp, rc) use med_utils_mod , only : chkerr => med_utils_ChkErr use med_methods_mod , only : FB_diagnose => med_methods_FB_diagnose use med_map_mod , only : med_map_field_packed - use med_fraction_mod , only : med_fraction_set + use med_fraction_mod , only : med_fraction_set use med_internalstate_mod , only : InternalState, mastertask use esmFlds , only : compice, compatm, compocn, compwav use perf_mod , only : t_startf, t_stopf diff --git a/mediator/med_phases_prep_glc_mod.F90 b/mediator/med_phases_prep_glc_mod.F90 index adff495d5..4c0879a2c 100644 --- a/mediator/med_phases_prep_glc_mod.F90 +++ b/mediator/med_phases_prep_glc_mod.F90 @@ -20,7 +20,7 @@ module med_phases_prep_glc_mod use ESMF , only : ESMF_FieldBundleCreate, ESMF_FieldBundleIsCreated use ESMF , only : ESMF_Field, ESMF_FieldGet, ESMF_FieldCreate use ESMF , only : ESMF_Mesh, ESMF_MESHLOC_ELEMENT, ESMF_TYPEKIND_R8, ESMF_KIND_R8 - use ESMF , only : ESMF_DYNAMICMASK, ESMF_DynamicMaskSetR8R8R8, ESMF_DYNAMICMASKELEMENTR8R8R8 + use ESMF , only : ESMF_DYNAMICMASK, ESMF_DynamicMaskSetR8R8R8, ESMF_DYNAMICMASKELEMENTR8R8R8 use ESMF , only : ESMF_FieldRegrid use esmFlds , only : complnd, compocn, mapbilnr, mapconsd, compname use esmFlds , only : max_icesheets, num_icesheets, compglc, ocn2glc_coupling @@ -106,7 +106,7 @@ module med_phases_prep_glc_mod type(ESMF_FieldBundle) :: FBocnAccum_o integer :: FBocnAccumCnt character(len=14) :: fldnames_fr_ocn(2) = (/'So_t_depth','So_s_depth'/) ! TODO: what else needs to be added here - type(ESMF_DynamicMask) :: dynamicOcnMask + type(ESMF_DynamicMask) :: dynamicOcnMask integer, parameter :: num_ocndepths = 7 logical :: ocn_sends_depths = .false. @@ -429,7 +429,7 @@ subroutine med_phases_prep_glc_init(gcomp, rc) end if end do - ! Create a dynamic mask object + ! Create a dynamic mask object ! The dynamic mask object further holds a pointer to the routine that will be called in order to ! handle dynamically masked elements - in this case its DynOcnMaskProc (see below) call ESMF_DynamicMaskSetR8R8R8(dynamicOcnMask, dynamicSrcMaskValue=czero, & @@ -450,10 +450,10 @@ subroutine med_phases_prep_glc_accum_lnd(gcomp, rc) !--------------------------------------- ! Carry out accumulation for the lnd->glc and ocn->glc - ! Accumulation and averaging is done on + ! Accumulation and averaging is done on ! - on the land mesh for land input ! - on the ocean mesh for ocean input - ! Mapping from the land to the glc grid and from the ocean to the glc grid + ! Mapping from the land to the glc grid and from the ocean to the glc grid ! is then done after the accumulated fields have been time averaged !--------------------------------------- @@ -497,7 +497,7 @@ subroutine med_phases_prep_glc_accum_lnd(gcomp, rc) do n = 1, size(fldnames_fr_lnd) call fldbun_getdata2d(is_local%wrap%FBImp(complnd,complnd), fldnames_fr_lnd(n), data2d_in, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call fldbun_getdata2d(FBlndAccum_l, fldnames_fr_lnd(n), data2d_out, rc) + call fldbun_getdata2d(FBlndAccum_l, fldnames_fr_lnd(n), data2d_out, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return do i = 1,size(data2d_out, dim=2) data2d_out(:,i) = data2d_out(:,i) + data2d_in(:,i) @@ -520,7 +520,7 @@ subroutine med_phases_prep_glc_accum_ocn(gcomp, rc) !--------------------------------------- ! Carry out accumulation for ocn->glc - ! Accumulation and averaging is done on + ! Accumulation and averaging is done on ! - on the ocean mesh for ocean input ! Mapping from from the ocean to the glc grid is then done after ! the accumulated fields have been time averaged @@ -567,7 +567,7 @@ subroutine med_phases_prep_glc_accum_ocn(gcomp, rc) do n = 1, size(fldnames_fr_ocn) call fldbun_getdata2d(is_local%wrap%FBImp(compocn,compocn), fldnames_fr_ocn(n), data2d_in, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call fldbun_getdata2d(FBocnAccum_o, fldnames_fr_ocn(n), data2d_out, rc) + call fldbun_getdata2d(FBocnAccum_o, fldnames_fr_ocn(n), data2d_out, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return do i = 1,size(data2d_out, dim=2) data2d_out(:,i) = data2d_out(:,i) + data2d_in(:,i) @@ -887,7 +887,7 @@ subroutine med_phases_prep_glc_map_lnd2glc(gcomp, rc) if (chkErr(rc,__LINE__,u_FILE_u)) return ! ------------------------------------------------------------------------ - ! Loop over fields in export field bundle to glc for ice sheet ns and + ! Loop over fields in export field bundle to glc for ice sheet ns and ! perform vertical interpolation of data onto ice sheet topography ! This maps all of the input elevation classes into an export to glc without elevation classes ! ------------------------------------------------------------------------ @@ -1076,7 +1076,7 @@ subroutine med_phases_prep_glc_renormalize_smb(gcomp, rc) real(r8) :: accum_renorm_factor ! ratio between global accumulation on the two grids real(r8) :: ablat_renorm_factor ! ratio between global ablation on the two grids real(r8) :: effective_area ! grid cell area multiplied by min(lfrac,icemask_l). - real(r8), pointer :: area_g(:) ! areas on glc grid + real(r8), pointer :: area_g(:) ! areas on glc grid character(len=*), parameter :: subname=' (renormalize_smb) ' !--------------------------------------------------------------- @@ -1257,7 +1257,7 @@ subroutine med_phases_prep_glc_renormalize_smb(gcomp, rc) end subroutine med_phases_prep_glc_renormalize_smb !================================================================================================ - subroutine dynOcnMaskProc(dynamicMaskList, dynamicSrcMaskValue, dynamicDstMaskValue, rc) + subroutine dynOcnMaskProc(dynamicMaskList, dynamicSrcMaskValue, dynamicDstMaskValue, rc) use ESMF, only : ESMF_RC_ARG_BAD diff --git a/mediator/med_phases_prep_ice_mod.F90 b/mediator/med_phases_prep_ice_mod.F90 index ae20593c5..7ec38e877 100644 --- a/mediator/med_phases_prep_ice_mod.F90 +++ b/mediator/med_phases_prep_ice_mod.F90 @@ -24,7 +24,7 @@ module med_phases_prep_ice_mod subroutine med_phases_prep_ice(gcomp, rc) use ESMF , only : operator(/=) - use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_StateGet + use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_StateGet use ESMF , only : ESMF_LogWrite, ESMF_LOGMSG_INFO, ESMF_SUCCESS use ESMF , only : ESMF_FieldBundleGet, ESMF_FieldGet, ESMF_Field use ESMF , only : ESMF_LOGMSG_ERROR, ESMF_FAILURE @@ -119,7 +119,7 @@ subroutine med_phases_prep_ice(gcomp, rc) ! obtain nextsw_cday from atm if it is in the import state and send it to ice scalar_id=is_local%wrap%flds_scalar_index_nextsw_cday if (scalar_id > 0 .and. mastertask) then - call ESMF_StateGet(is_local%wrap%NstateImp(compatm), & + call ESMF_StateGet(is_local%wrap%NstateImp(compatm), & itemName=trim(is_local%wrap%flds_scalar_name), field=lfield, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return call ESMF_FieldGet(lfield, farrayPtr=dataptr_scalar_atm, rc=rc) diff --git a/mediator/med_phases_prep_lnd_mod.F90 b/mediator/med_phases_prep_lnd_mod.F90 index 179b61b43..ca1ed38d5 100644 --- a/mediator/med_phases_prep_lnd_mod.F90 +++ b/mediator/med_phases_prep_lnd_mod.F90 @@ -94,7 +94,7 @@ subroutine med_phases_prep_lnd(gcomp, rc) ! obtain nextsw_cday from atm if it is in the import state and send it to lnd scalar_id=is_local%wrap%flds_scalar_index_nextsw_cday if (scalar_id > 0 .and. mastertask) then - call ESMF_StateGet(is_local%wrap%NstateImp(compatm), & + call ESMF_StateGet(is_local%wrap%NstateImp(compatm), & itemName=trim(is_local%wrap%flds_scalar_name), field=lfield, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return call ESMF_FieldGet(lfield, farrayPtr=dataptr_scalar_atm, rc=rc) From 02ceba38f590c829ea5b83476077f2fb0210dadc Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 24 Mar 2021 12:39:35 -0400 Subject: [PATCH 37/42] add missing dststatusfield for mappatch and mappatch_uv3d --- mediator/med_map_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index b6fbd2220..0a8ab3580 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -411,6 +411,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, polemethod=polemethod, & srcTermProcessing=srcTermProcessing_Value, & ignoreDegenerate=.true., & + dstStatusField=dststatusfield, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return end if From 9a0cad5f6f667ac8c222507f1a62d9341398243a Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 24 Mar 2021 16:41:45 -0600 Subject: [PATCH 38/42] optionally read dststatus_print from MED_attributes --- mediator/esmFlds.F90 | 2 ++ mediator/med.F90 | 10 +++++++++- mediator/med_map_mod.F90 | 3 +-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mediator/esmFlds.F90 b/mediator/esmFlds.F90 index 9a937ebfa..185d52096 100644 --- a/mediator/esmFlds.F90 +++ b/mediator/esmFlds.F90 @@ -34,6 +34,8 @@ module esmflds integer, public :: num_icesheets = 1 logical, public :: ocn2glc_coupling ! obtained from attribute + logical, public :: dststatus_print = .false. + !----------------------------------------------- ! Set mappers !----------------------------------------------- diff --git a/mediator/med.F90 b/mediator/med.F90 index 06a2e61af..fa157fe45 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -557,7 +557,8 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc) use ESMF , only : ESMF_LogWrite, ESMF_LOGMSG_INFO, ESMF_METHOD_INITIALIZE use NUOPC , only : NUOPC_CompFilterPhaseMap, NUOPC_CompAttributeGet use med_internalstate_mod, only : mastertask, logunit - + use esmFlds, only : dststatus_print + type(ESMF_GridComp) :: gcomp type(ESMF_State) :: importState, exportState type(ESMF_Clock) :: clock @@ -626,6 +627,13 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc) write(msgString,'(A,i6)') trim(subname)//': Mediator dbug_flag is ',dbug_flag call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + ! Obtain dststatus_print setting if present; otherwise use default value in med_constants + call NUOPC_CompAttributeGet(gcomp, name='dststatus_print', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) dststatus_print=(trim(cvalue)=="true") + write(msgString,*) trim(subname)//': Mediator dststatus_print is ',dststatus_print + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + ! Switch to IPDv03 by filtering all other phaseMap entries call NUOPC_CompFilterPhaseMap(gcomp, ESMF_METHOD_INITIALIZE, acceptStringList=(/"IPDv03p"/), rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 0a8ab3580..f9baa6033 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -222,7 +222,7 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, use esmFlds , only : mapfillv_bilnr, mapbilnr_nstod use esmFlds , only : ncomps, compatm, compice, compocn, compname use esmFlds , only : mapfcopy, mapconsd, mapconsf, mapnstod - use esmFlds , only : coupling_mode + use esmFlds , only : coupling_mode, dststatus_print use esmFlds , only : atm_name use med_constants_mod , only : ispval_mask => med_constants_ispval_mask @@ -247,7 +247,6 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, integer :: dstMaskValue character(len=ESMF_MAXSTR) :: lmapfile logical :: rhprint = .false. - logical :: dststatus_print = .false. integer :: ns integer(I4), pointer :: dof(:) => null() integer :: srcTermProcessing_Value = 0 From c6a10fd971b98a38b3cfb3bd2879302d5d38dbb1 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 24 Mar 2021 16:43:35 -0600 Subject: [PATCH 39/42] clean up comment --- mediator/med.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediator/med.F90 b/mediator/med.F90 index fa157fe45..74073e86a 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -627,7 +627,7 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc) write(msgString,'(A,i6)') trim(subname)//': Mediator dbug_flag is ',dbug_flag call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) - ! Obtain dststatus_print setting if present; otherwise use default value in med_constants + ! Obtain dststatus_print setting if present call NUOPC_CompAttributeGet(gcomp, name='dststatus_print', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (isPresent .and. isSet) dststatus_print=(trim(cvalue)=="true") From 8522323e7285041f374ab18b7e369865104eeb10 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 24 Mar 2021 16:51:06 -0600 Subject: [PATCH 40/42] Remove valid_values for GLC_GRID This way we don't need to change this xml entry whenever we want to add a new GLC grid. None of the other *_GRID variables define valid_values. Also, set default_value to UNSET, as for other *_GRID variables. --- cime_config/config_component.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 34e2d1965..4c1686b7b 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1177,8 +1177,7 @@ char - gland20,gland10,gland5,gland5UM,gland4,mpas.gis20km,mpas.ais20km,null - gland5UM + UNSET build_grid env_build.xml glacier (glc) grid - DO NOT EDIT (for experts only) From b4371486682a9a57bf5597fe03fedfad8f599749 Mon Sep 17 00:00:00 2001 From: mvertens Date: Sat, 27 Mar 2021 13:48:47 -0600 Subject: [PATCH 41/42] update for getting the glc present flag set correctly with nag compiler --- mediator/med.F90 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mediator/med.F90 b/mediator/med.F90 index ecc17851d..66f18bfe5 100644 --- a/mediator/med.F90 +++ b/mediator/med.F90 @@ -842,9 +842,9 @@ subroutine InitializeIPDv03p1(gcomp, importState, exportState, clock, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call NUOPC_CompAttributeSet(gcomp, name="rof_present", value=rof_present, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompAttributeSet(gcomp, name="wav_present", value=wav_present, rc=rc) + call NUOPC_CompAttributeSet(gcomp, name="wav_present", value=trim(wav_present), rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompAttributeSet(gcomp, name="glc_present", value=glc_present, rc=rc) + call NUOPC_CompAttributeSet(gcomp, name="glc_present", value=trim(glc_present), rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call NUOPC_CompAttributeSet(gcomp, name="med_present", value=med_present, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -1790,15 +1790,15 @@ subroutine DataInitialize(gcomp, rc) call ESMF_AttributeGet(gcomp, name="glc_present", value=cvalue, & convention="NUOPC", purpose="Instance", rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - if (trim(cvalue) == 'true') then - do ns = 1,max_icesheets - if (ns <= num_icesheets) then + do ns = 1,max_icesheets + if (ns <= num_icesheets) then + if (trim(cvalue) == 'true') then is_local%wrap%comp_present(compglc(ns)) = .true. else is_local%wrap%comp_present(compglc(ns)) = .false. end if - end do - end if + end if + end do else call ESMF_AttributeGet(gcomp, name=trim(compname(n1))//"_present", value=cvalue, & convention="NUOPC", purpose="Instance", rc=rc) From 4556f5e10f489dc7c97e1856e52934642906e578 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Sun, 28 Mar 2021 19:30:02 +0000 Subject: [PATCH 42/42] revert change srcMaskValue * datm will remain nemsdatm until cdeps is added so revert change to srcMaskValue for compatm = "datm" --- mediator/med_map_mod.F90 | 3 --- 1 file changed, 3 deletions(-) diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90 index 1690eb270..f9baa6033 100644 --- a/mediator/med_map_mod.F90 +++ b/mediator/med_map_mod.F90 @@ -277,9 +277,6 @@ subroutine med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, if (n1 == compatm .and. (n2 == compocn .or. n2 == compice)) then srcMaskValue = 1 dstMaskValue = 0 - if (trim(atm_name) == 'datm') then - srcMaskValue = 0 - endif else if (n2 == compatm .and. (n1 == compocn .or. n1 == compice)) then srcMaskValue = 0 dstMaskValue = 1