From 32e9a63c5f772f3ef91868f1f73f2e0521067540 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Sun, 3 May 2020 18:07:35 +0000 Subject: [PATCH 01/14] fix clock issue --- DATM/datm.F90 | 103 ++++++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 63 deletions(-) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index c54bc9b..f3e3f5d 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -225,7 +225,6 @@ subroutine InitializeP1(model, importState, exportState, externalClock, rc) write(msgString,'(a,i6)')'Model configure found with nfhout = ',nfhout call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ConfigGetAttribute(config=cf, & value=filename_base, & label='filename_base:',rc=rc) @@ -395,20 +394,24 @@ subroutine ModelAdvance(model, rc) currTime=currTime, & startTime=startTime, & stopTime=stopTime, & + timeStep=timeStep, & rc=rc) - call ESMF_ClockPrint(modelClock, options="currTime", & + call ESMF_TimePrint(currTime, & preString="ModelAdvance DATM with CLOCK current: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(modelClock, options="stopTime", & + call ESMF_TimePrint(currTime + timeStep, & + preString="ModelAdvance DATM with CLOCK advance to: ", & + unit=msgString, rc=rc) + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + call ESMF_TimePrint(stopTime, & preString="ModelAdvance DATM with CLOCK stop: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockGet(modelClock, currTime=currTime, timeStep=timeStep, rc=rc) - ! TODO: why is this the export time? - call ESMF_TimeGet(currTime, timestring=export_timestr, rc=rc) + call ESMF_TimeGet(currTime+timestep, timestring=export_timestr, rc=rc) + call ESMF_LogWrite(trim(export_timestr), ESMF_LOGMSG_INFO) ! Run the component call AtmRun(model, importState, exportState, modelClock, rc) @@ -426,39 +429,6 @@ subroutine ModelAdvance(model, rc) end subroutine ModelAdvance - ! like cice_cap, which had the simplest clock settings I could find - !----------------------------------------------------------------------------- - - subroutine SetClock(model, rc) - - type(ESMF_GridComp) :: model - integer, intent(out) :: rc - - ! local variables - type(ESMF_Clock) :: clock - type(ESMF_TimeInterval) :: stabilityTimeStep, timeStep - - rc = ESMF_SUCCESS - - ! query the Component for its clock - call ESMF_GridCompGet(model, clock=clock, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_TimeIntervalSet(timestep, s_r8=dt_atmos, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_ClockSet(clock, timestep=timestep, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - ! initialize internal clock - ! here: parent Clock and stability timeStep determine actual model timeStep - call ESMF_TimeIntervalSet(stabilityTimeStep, s_r8=dt_atmos, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompSetClock(model, clock, stabilityTimeStep, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - end subroutine SetClock - !----------------------------------------------------------------------------- subroutine SetRunClock(model, rc) @@ -468,10 +438,13 @@ subroutine SetRunClock(model, rc) ! local variables type(ESMF_Clock) :: modelClock, driverClock - type(ESMF_Time) :: currTime - type(ESMF_TimeInterval) :: timeStep + type(ESMF_Time) :: mcurrtime, dcurrtime + type(ESMF_Time) :: mstoptime, dstoptime + type(ESMF_TimeInterval) :: mtimestep, dtimestep + character(len=ESMF_MAXSTR) :: mtimestring, dtimestring character(len=ESMF_MAXSTR) :: msgString + character(len=ESMF_MAXSTR) :: subname = "SetRunClock" rc = ESMF_SUCCESS #ifndef coupled @@ -485,36 +458,40 @@ subroutine SetRunClock(model, rc) driverClock=driverClock, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - ! set the modelClock to have the current start time as the driverClock - call ESMF_ClockGet(driverClock, currTime=currTime, timeStep=timeStep, rc=rc) + call ESMF_ClockGet(driverClock, currTime=dcurrtime, timeStep=dtimestep, & + stopTime=dstoptime, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_TimeIntervalSet(timestep, s_r8=dt_atmos, rc=rc) + call ESMF_ClockGet(modelClock, currTime=mcurrtime, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_ClockSet(modelClock, currTime=currTime, timeStep=timestep, rc=rc) + call ESMF_TimeIntervalSet(mtimestep, s_r8=dt_atmos, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_ClockPrint(modelClock, options="currTime", & - preString="entering SetRunClock DATM with modelClock current: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(driverClock, options="currTime", & - preString="entering SetRunClock DATM with driverClock current: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) + !-------------------------------- + ! check that the current time in the model and driver are the same + !-------------------------------- - call ESMF_ClockPrint(modelClock, options="stopTime", & - preString="entering SetRunClock DATM with modelClock stop: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(driverClock, options="stopTime", & - preString="entering SetRunClock DATM with driverClock stop: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) + if (mcurrtime /= dcurrtime) then + call ESMF_TimeGet(dcurrtime, timeString=dtimestring, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_TimeGet(mcurrtime, timeString=mtimestring, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_LogSetError(ESMF_RC_VAL_WRONG, & + msg=trim(subname)//": ERROR in time consistency: "//trim(dtimestring)//" != "//trim(mtimestring), & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + + !-------------------------------- + ! force model clock currtime and timestep to match driver and set stoptime + !-------------------------------- + + mstoptime = mcurrtime + dtimestep - ! check and set the component clock against the driver clock - call NUOPC_CompCheckSetClock(model, driverClock, rc=rc) + call ESMF_ClockSet(modelClock, currTime=dcurrtime, timeStep=dtimestep, stopTime=mstoptime, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_LogWrite("User routine SetRunClock Atm finished", ESMF_LOGMSG_INFO) From e7edc74d1b3b8bffcf1d8a43886fbe1250ff74fb Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Tue, 5 May 2020 11:14:20 +0000 Subject: [PATCH 02/14] remove unused importState where possible --- DATM/AtmBundleCreate.F90 | 3 +-- DATM/AtmFieldUtils.F90 | 10 ++++------ DATM/AtmForce.F90 | 2 +- DATM/AtmModel.F90 | 10 ++++------ DATM/datm.F90 | 27 ++++++++++++++------------- 5 files changed, 24 insertions(+), 28 deletions(-) diff --git a/DATM/AtmBundleCreate.F90 b/DATM/AtmBundleCreate.F90 index c830dc6..0184ec3 100644 --- a/DATM/AtmBundleCreate.F90 +++ b/DATM/AtmBundleCreate.F90 @@ -1,4 +1,4 @@ -subroutine AtmBundleCreate(gcomp,importState, exportState, rc) +subroutine AtmBundleCreate(gcomp, exportState, rc) use ESMF @@ -7,7 +7,6 @@ subroutine AtmBundleCreate(gcomp,importState, exportState, rc) implicit none type(ESMF_GridComp) :: gcomp - type(ESMF_State) :: importState type(ESMF_State) :: exportState integer, intent(out) :: rc diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index 18dce2f..e879635 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -115,11 +115,11 @@ end subroutine AtmFieldsRealize !------------------------------------------------------------------------------------- - subroutine AtmFieldCheck(importState, exportState, tag, rc) + subroutine AtmFieldCheck(exportState, tag, rc) use AtmInternalFields, only : iprnt, jprnt - type(ESMF_State) :: importState, exportState + type(ESMF_State) :: exportState character(len=*), intent( in) :: tag integer, intent(out) :: rc @@ -170,9 +170,8 @@ end subroutine AtmFieldCheck !----------------------------------------------------------------------------- - subroutine AtmFieldDump(importState, exportState, tag, timestr, rc) + subroutine AtmFieldDump(exportState, tag, timestr, rc) - type(ESMF_State) :: importState type(ESMF_State) :: exportState character(len=*), intent( in) :: tag character(len=*), intent( in) :: timestr @@ -325,7 +324,7 @@ end subroutine AtmBundleCheck !----------------------------------------------------------------------------- - subroutine AtmBundleIntp(gcomp, importState, exportState, externalClock, hour, rc) + subroutine AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) use AtmInternalFields, only : hfwd,hbak use AtmInternalFields, only : iprnt,jprnt @@ -333,7 +332,6 @@ subroutine AtmBundleIntp(gcomp, importState, exportState, externalClock, hour, r use AtmInternalFields, only : AtmBundleFwd, AtmBundleBak type(ESMF_GridComp) :: gcomp - type(ESMF_State) :: importState type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock diff --git a/DATM/AtmForce.F90 b/DATM/AtmForce.F90 index 214d6b1..0f2bc3b 100644 --- a/DATM/AtmForce.F90 +++ b/DATM/AtmForce.F90 @@ -76,7 +76,7 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) call ESMF_TimeGet(nextTime,h_r8=hfwd,rc=rc) endif - write(msgString,'(3a,f12.3)')'using ',trim(filename),' at fwd clock hour ',real(hfwd,4) + write(msgString,'(3a,f12.3)')'using ',trim(filename),' at fwd clock hour ',hfwd call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) ! read the Atm field data into the Fwd bundle diff --git a/DATM/AtmModel.F90 b/DATM/AtmModel.F90 index 87d1a32..475ade3 100644 --- a/DATM/AtmModel.F90 +++ b/DATM/AtmModel.F90 @@ -20,12 +20,11 @@ module AtmModel contains - subroutine AtmInit(gcomp, importState, exportState, externalClock, rc) + subroutine AtmInit(gcomp, exportState, externalClock, rc) use AtmInternalFields type(ESMF_GridComp) :: gcomp - type(ESMF_State) :: importState type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc @@ -82,17 +81,16 @@ subroutine AtmInit(gcomp, importState, exportState, externalClock, rc) call AtmForce(gcomp,exportState,externalClock,0,rc) call AtmBundleCheck('after AtmForce',rc) !time interpolate between fwd & bak values - call AtmBundleIntp(gcomp, importState, exportState, externalClock, 0.0d8, rc) + call AtmBundleIntp(gcomp, exportState, externalClock, 0.0d8, rc) call ESMF_LogWrite("User run routine AtmInit finished", ESMF_LOGMSG_INFO) end subroutine AtmInit !----------------------------------------------------------------------------- - subroutine AtmRun(gcomp, importState, exportState, externalClock, rc) + subroutine AtmRun(gcomp, exportState, externalClock, rc) type(ESMF_GridComp) :: gcomp - type(ESMF_State) :: importState type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc @@ -140,7 +138,7 @@ subroutine AtmRun(gcomp, importState, exportState, externalClock, rc) call AtmBundleCheck('after AtmRun',rc) !time interpolate between fwd & bak values - call AtmBundleIntp(gcomp, importState, exportState, externalClock, hour, rc) + call AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) call ESMF_LogWrite("User run routine AtmRun finished", ESMF_LOGMSG_INFO) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index f3e3f5d..dfa1538 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -106,7 +106,8 @@ end subroutine SetServices subroutine InitializeP0(model, importState, exportState, externalClock, rc) type(ESMF_GridComp) :: model - type(ESMF_State) :: importState, exportState + type(ESMF_State) :: importState + type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc @@ -169,7 +170,8 @@ end subroutine InitializeP0 subroutine InitializeP1(model, importState, exportState, externalClock, rc) type(ESMF_GridComp) :: model - type(ESMF_State) :: importState, exportState + type(ESMF_State) :: importState + type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc @@ -258,7 +260,8 @@ end subroutine InitializeP1 subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_GridComp) :: model - type(ESMF_State) :: importState, exportState + type(ESMF_State) :: importState + type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc @@ -312,13 +315,13 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) !call ESMF_GridCompPrint(model, rc=rc) ! Create and fill the AtmBundle - call AtmBundleCreate(model, importState, exportState, rc) + call AtmBundleCreate(model, exportState, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call AtmFieldsRealize(exportState, gridOut, AtmBundleFields, 'Atm Export', rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call AtmInit(model, importState, exportState, externalClock, rc) + call AtmInit(model, exportState, externalClock, rc) ! AtmInit calls AtmForce and loads the values for the first integration ! timestep, so..... @@ -348,7 +351,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_LogWrite('Atm InitializeDataComplete', ESMF_LOGMSG_INFO) - call AtmFieldCheck(importState, exportState, 'InitP2 Atm', rc) + call AtmFieldCheck(exportState, 'InitP2 Atm', rc) call ESMF_LogWrite("User initialize routine InitP2 Atm finished", ESMF_LOGMSG_INFO) @@ -362,7 +365,7 @@ subroutine ModelAdvance(model, rc) integer, intent(out) :: rc ! local variables - type(ESMF_State) :: importState, exportState + type(ESMF_State) :: exportState type(ESMF_Clock) :: modelClock type(ESMF_Time) :: stopTime type(ESMF_Time) :: startTime @@ -376,10 +379,9 @@ subroutine ModelAdvance(model, rc) call ESMF_LogWrite("User routine ModelAdvance Atm started", ESMF_LOGMSG_INFO) - ! query the Component for its clock, importState and exportState + ! query the Component for its clock and exportState call NUOPC_ModelGet(model, & modelClock=modelClock, & - importState=importState, & exportState=exportState, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -411,17 +413,16 @@ subroutine ModelAdvance(model, rc) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) call ESMF_TimeGet(currTime+timestep, timestring=export_timestr, rc=rc) - call ESMF_LogWrite(trim(export_timestr), ESMF_LOGMSG_INFO) ! Run the component - call AtmRun(model, importState, exportState, modelClock, rc) + call AtmRun(model, exportState, modelClock, rc) ! Check Values - call AtmFieldCheck(importState, exportState, 'after AtmRun', rc) + call AtmFieldCheck(exportState, 'after AtmRun', rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if(dumpfields)then - call AtmFieldDump(importstate, exportstate, 'after AtmRun', trim(export_timestr), rc) + call AtmFieldDump(exportstate, 'after AtmRun', trim(export_timestr), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif From d1dea41b671e07574364e0626772eb66695096f7 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Fri, 8 May 2020 16:47:32 +0000 Subject: [PATCH 03/14] changed capitalization --- DATM/datm.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index dfa1538..59471c7 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -422,7 +422,7 @@ subroutine ModelAdvance(model, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if(dumpfields)then - call AtmFieldDump(exportstate, 'after AtmRun', trim(export_timestr), rc) + call AtmFieldDump(exportState, 'after AtmRun', trim(export_timestr), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif From 7aadddb26b6cdd07eb95cf700b6681215d319535 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Fri, 8 May 2020 12:57:51 -0400 Subject: [PATCH 04/14] add dumping of fields at initialization --- DATM/AtmFieldUtils.F90 | 1 + DATM/datm.F90 | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index e879635..9c4b3ec 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -190,6 +190,7 @@ subroutine AtmFieldDump(exportState, tag, timestr, rc) call ESMF_LogWrite("User routine AtmFieldDump started", ESMF_LOGMSG_INFO) + ! Atm variables in exportState nfields = size(AtmBundleFields) do ii = 1,nfields call ESMF_StateGet(exportState, & diff --git a/DATM/datm.F90 b/DATM/datm.F90 index 59471c7..1d53f7d 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -263,6 +263,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_State) :: importState type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock + type(ESMF_Time) :: currTime integer, intent(out) :: rc ! local variables @@ -270,6 +271,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_Grid) :: gridOut type(ESMF_Field) :: field character(ESMF_MAXSTR) :: msgString + character(ESMF_MAXSTR) :: export_timestr character(ESMF_MAXSTR) :: fname integer :: ii, nfields @@ -353,6 +355,15 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) call AtmFieldCheck(exportState, 'InitP2 Atm', rc) + ! the initial fields at model startup + if(dumpfields)then + call ESMF_ClockGet(externalClock, currTime=currTime, rc = rc) + call ESMF_TimeGet(currTime, timestring=export_timestr, rc=rc) + + call AtmFieldDump(exportState, 'after AtmRun', trim(export_timestr), rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + endif + call ESMF_LogWrite("User initialize routine InitP2 Atm finished", ESMF_LOGMSG_INFO) end subroutine InitializeP2 From 4d343d085ce63406d32aa38978ac776c6024179f Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Mon, 11 May 2020 12:21:01 -0600 Subject: [PATCH 05/14] add check for missing settings in datm_data_table --- DATM/AtmInternalFields.F90 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DATM/AtmInternalFields.F90 b/DATM/AtmInternalFields.F90 index c10f70a..a7e48ca 100644 --- a/DATM/AtmInternalFields.F90 +++ b/DATM/AtmInternalFields.F90 @@ -62,6 +62,7 @@ module AtmInternalFields integer, public :: lPet, petCnt ! a diagnostic point to print at integer, public :: iprnt, jprnt + integer :: icnt ! called by AtmInit public :: AtmBundleSetUp @@ -325,13 +326,19 @@ subroutine AtmBundleSetUp if (ChkErr(rc,__LINE__,u_FILE_u)) return nfields = size(AtmBundleFields) + icnt = 0 do ii = 1,nfields call ESMF_ConfigGetAttribute(config=cfdata, & value=lvalue, & label=trim(AtmBundleFields(ii)%standard_name),rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return AtmBundleFields(ii)%isPresent=lvalue + icnt = icnt + 1 enddo + if(icnt .ne. nfields)then + call ESMF_LogWrite('Missing input fields in datm_data_table', ESMF_LOGMSG_INFO) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif !----------------------------------------------------------------------------- ! check From 21af15fc7f5a8dcef7da2340cafa73c4fa145257 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 13 May 2020 11:20:39 +0000 Subject: [PATCH 06/14] send file name to field dump routine --- DATM/AtmFieldUtils.F90 | 14 ++------------ DATM/datm.F90 | 13 ++++++++----- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index 9c4b3ec..5b4d5d2 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -170,11 +170,10 @@ end subroutine AtmFieldCheck !----------------------------------------------------------------------------- - subroutine AtmFieldDump(exportState, tag, timestr, rc) + subroutine AtmFieldDump(exportState, filename, rc) type(ESMF_State) :: exportState - character(len=*), intent( in) :: tag - character(len=*), intent( in) :: timestr + character(len=*), intent( in) :: filename integer, intent(out) :: rc ! Local variables @@ -182,7 +181,6 @@ subroutine AtmFieldDump(exportState, tag, timestr, rc) integer :: ii,nfields character(len=ESMF_MAXSTR) :: varname - character(len=ESMF_MAXSTR) :: filename character(len=ESMF_MAXSTR) :: msgString ! Initialize return code @@ -198,9 +196,6 @@ subroutine AtmFieldDump(exportState, tag, timestr, rc) field=field,rc=rc) varname = trim(AtmBundleFields(ii)%shortname) - if(trim(tag) .eq. 'before AtmRun')filename = 'field_atm_exportb_'//trim(timestr)//'.nc' - if(trim(tag) .eq. 'after AtmRun')filename = 'field_atm_exporta_'//trim(timestr)//'.nc' - write(msgString, '(a,i6)')'Writing exportState field '//trim(varname)//' to ' & //trim(filename) call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) @@ -209,11 +204,6 @@ subroutine AtmFieldDump(exportState, tag, timestr, rc) fileName =trim(filename), & timeslice=1, & overwrite=.true., rc=rc) - !endif - !if(iicnt .eq. 1)then - ! write(msgString, *)'Writing exportState field ',trim(varname),' to ',trim(filename) - ! call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) - !endif enddo call ESMF_LogWrite("User routine AtmFieldDump finished", ESMF_LOGMSG_INFO) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index 1d53f7d..da2871e 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -263,7 +263,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_State) :: importState type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock - type(ESMF_Time) :: currTime + type(ESMF_Time) :: currTime integer, intent(out) :: rc ! local variables @@ -271,7 +271,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_Grid) :: gridOut type(ESMF_Field) :: field character(ESMF_MAXSTR) :: msgString - character(ESMF_MAXSTR) :: export_timestr + character(ESMF_MAXSTR) :: timestr character(ESMF_MAXSTR) :: fname integer :: ii, nfields @@ -358,9 +358,10 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) ! the initial fields at model startup if(dumpfields)then call ESMF_ClockGet(externalClock, currTime=currTime, rc = rc) - call ESMF_TimeGet(currTime, timestring=export_timestr, rc=rc) + call ESMF_TimeGet(currTime, timestring=timestr, rc=rc) - call AtmFieldDump(exportState, 'after AtmRun', trim(export_timestr), rc) + fname = 'field_atm_exporta_init_'//trim(timestr) + call AtmFieldDump(exportState, trim(fname), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif @@ -383,6 +384,7 @@ subroutine ModelAdvance(model, rc) type(ESMF_Time) :: currTime type(ESMF_TimeInterval) :: timeStep + character(len=ESMF_MAXSTR) :: fname character(len=ESMF_MAXSTR) :: msgString character(len=ESMF_MAXSTR) :: export_timestr @@ -433,7 +435,8 @@ subroutine ModelAdvance(model, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if(dumpfields)then - call AtmFieldDump(exportState, 'after AtmRun', trim(export_timestr), rc) + fname = 'field_atm_exporta_'//trim(export_timestr) + call AtmFieldDump(exportState, trim(fname), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif From 05cc1cad655ebfa062f3563fa95493563d9392b2 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 13 May 2020 13:35:51 +0000 Subject: [PATCH 07/14] add dbug_flag setting read from nems.configure to control output --- DATM/AtmFieldUtils.F90 | 14 ++++++++------ DATM/AtmInternalFields.F90 | 2 +- DATM/AtmModel.F90 | 9 +++++---- DATM/datm.F90 | 11 +++++++++-- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index 5b4d5d2..d4b18b0 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -393,12 +393,14 @@ subroutine AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) ! real(AtmBundleFields(ii)%farrayPtr_fwd(ijloc(1),ijloc(2)),4) !call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) - write (msgString,*)' AtmIntp ',& - trim(AtmBundleFields(ii)%shortname),& - real(AtmBundleFields(ii)%farrayPtr(iprnt,jprnt),4),& - real(AtmBundleFields(ii)%farrayPtr_bak(iprnt,jprnt),4),& - real(AtmBundleFields(ii)%farrayPtr_fwd(iprnt,jprnt),4) - call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + if(dbug_flag > 0)then + write (msgString,*)' AtmIntp ',& + trim(AtmBundleFields(ii)%shortname),& + real(AtmBundleFields(ii)%farrayPtr(iprnt,jprnt),4),& + real(AtmBundleFields(ii)%farrayPtr_bak(iprnt,jprnt),4),& + real(AtmBundleFields(ii)%farrayPtr_fwd(iprnt,jprnt),4) + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + end if !write (msgString,*)' AtmBundleFields ',& ! trim(AtmBundleFields(ii)%field_name),' min,max,sum ',& diff --git a/DATM/AtmInternalFields.F90 b/DATM/AtmInternalFields.F90 index a7e48ca..a244f82 100644 --- a/DATM/AtmInternalFields.F90 +++ b/DATM/AtmInternalFields.F90 @@ -59,7 +59,7 @@ module AtmInternalFields type(AtmField_Definition), public :: AtmBundleFields(AtmFieldCount) - integer, public :: lPet, petCnt + integer, public :: lPet, petCnt, dbug_flag ! a diagnostic point to print at integer, public :: iprnt, jprnt integer :: icnt diff --git a/DATM/AtmModel.F90 b/DATM/AtmModel.F90 index 475ade3..c04b317 100644 --- a/DATM/AtmModel.F90 +++ b/DATM/AtmModel.F90 @@ -4,6 +4,7 @@ module AtmModel use ESMF use AtmInternalFields, only : ChkErr + use AtmInternalFields, only : dbug_flag use AtmInternalFields, only : hfwd, hbak, nfhout use AtmFieldUtils, only : AtmForceFwd2Bak, AtmBundleCheck use AtmFieldUtils, only : AtmBundleIntp @@ -72,14 +73,14 @@ subroutine AtmInit(gcomp, exportState, externalClock, rc) ! initialize the fwd and bak fields as special case at initialzation call AtmForce(gcomp,exportState,externalClock,0,rc) - call AtmBundleCheck('after AtmForce',rc) + if(dbug_flag > 5)call AtmBundleCheck('after AtmForce',rc) ! copy the fwd timestamp to the bak timestamp hbak = hfwd call AtmForceFwd2Bak(rc) - call AtmBundleCheck('after Fwd2Bak',rc) + if(dbug_flag > 5)call AtmBundleCheck('after Fwd2Bak',rc) ! reload the fwd call AtmForce(gcomp,exportState,externalClock,0,rc) - call AtmBundleCheck('after AtmForce',rc) + if(dbug_flag > 5)call AtmBundleCheck('after AtmForce',rc) !time interpolate between fwd & bak values call AtmBundleIntp(gcomp, exportState, externalClock, 0.0d8, rc) @@ -136,7 +137,7 @@ subroutine AtmRun(gcomp, exportState, externalClock, rc) write(msgString,*)'AtmRun: hbkd,hour,hfwd ', hbak,hour,hfwd call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO, rc=rc) - call AtmBundleCheck('after AtmRun',rc) + if(dbug_flag > 5)call AtmBundleCheck('after AtmRun',rc) !time interpolate between fwd & bak values call AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index da2871e..ed82b27 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -179,6 +179,7 @@ subroutine InitializeP1(model, importState, exportState, externalClock, rc) type(ESMF_Config) :: cf real(ESMF_KIND_R8) :: medAtmCouplingIntervalSec character(ESMF_MAXSTR) :: msgString + character(20) :: cvalue rc = ESMF_SUCCESS @@ -251,6 +252,12 @@ subroutine InitializeP1(model, importState, exportState, externalClock, rc) medAtmCouplingIntervalSec call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + call NUOPC_CompAttributeGet(model, name='DebugFlag', value=cvalue, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + read(cvalue,*) dbug_flag + write(msgString,'(a,i6)')'DebugFlag = ',dbug_flag + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + call ESMF_LogWrite("User initialize routine InitP1 Atm finished", ESMF_LOGMSG_INFO) end subroutine InitializeP1 @@ -353,7 +360,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_LogWrite('Atm InitializeDataComplete', ESMF_LOGMSG_INFO) - call AtmFieldCheck(exportState, 'InitP2 Atm', rc) + if(dbug_flag > 5)call AtmFieldCheck(exportState, 'InitP2 Atm', rc) ! the initial fields at model startup if(dumpfields)then @@ -431,7 +438,7 @@ subroutine ModelAdvance(model, rc) call AtmRun(model, exportState, modelClock, rc) ! Check Values - call AtmFieldCheck(exportState, 'after AtmRun', rc) + if(dbug_flag > 5)call AtmFieldCheck(exportState, 'after AtmRun', rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if(dumpfields)then From 4a0ecaf2d22a09fa86f0097c0481618e4a5b801f Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 13 May 2020 17:15:30 +0000 Subject: [PATCH 08/14] remove init part of name --- DATM/datm.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index ed82b27..c2d80ed 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -367,7 +367,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) call ESMF_ClockGet(externalClock, currTime=currTime, rc = rc) call ESMF_TimeGet(currTime, timestring=timestr, rc=rc) - fname = 'field_atm_exporta_init_'//trim(timestr) + fname = 'field_atm_exporta_'//trim(timestr)//'.nc' call AtmFieldDump(exportState, trim(fname), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif @@ -442,7 +442,7 @@ subroutine ModelAdvance(model, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if(dumpfields)then - fname = 'field_atm_exporta_'//trim(export_timestr) + fname = 'field_atm_exporta_'//trim(export_timestr)//'.nc' call AtmFieldDump(exportState, trim(fname), rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return endif From 15ed227a571ef41421a9b253f5d312be63e2f4a6 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 13 May 2020 18:00:21 -0600 Subject: [PATCH 09/14] adding scalars for cmeps --- DATM/AtmFieldUtils.F90 | 60 +++++++++++++++++++++++++++++++++++++- DATM/AtmInternalFields.F90 | 4 +++ DATM/datm.F90 | 50 +++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index d4b18b0..e59abc8 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -69,17 +69,26 @@ subroutine AtmFieldsRealize(state, grid, field_defs, tag, rc) character(len=*), intent(in ) :: tag integer, intent(out ) :: rc + integer :: localPet + type(ESMF_VM) :: vm type(ESMF_ArraySpec) :: arrayspecR8 type(ESMF_Field) :: field type(ESMF_StaggerLoc) :: staggerloc character(len=ESMF_MAXSTR) :: msgString - + integer :: mpicom + integer :: npet integer :: ii,nfields logical :: connected rc = ESMF_SUCCESS call ESMF_LogWrite("User routine AtmFieldsRealize "//trim(tag)//" started", ESMF_LOGMSG_INFO) + call ESMF_VMGetCurrent(vm, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_VMGet(vm, petCount=npet, mpiCommunicator=mpicom, localPet=localPet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + nfields=size(field_defs) !print *,'found nfields = ',nfields,' to realize ',field_defs%field_name @@ -110,6 +119,20 @@ subroutine AtmFieldsRealize(state, grid, field_defs, tag, rc) !enddo enddo + !--------------------------------- + ! set scalar data in export state + !--------------------------------- + + if (len_trim(scalar_field_name) > 0) then + call State_SetScalar(dble(iatm),scalar_field_idx_grid_nx, state, localPet, & + scalar_field_name, scalar_field_count, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call State_SetScalar(dble(jatm),scalar_field_idx_grid_ny, state, localPet, & + scalar_field_name, scalar_field_count, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + endif + call ESMF_LogWrite("User routine AtmFieldsRealize "//trim(tag)//" finished", ESMF_LOGMSG_INFO) end subroutine AtmFieldsRealize @@ -413,4 +436,39 @@ subroutine AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) call ESMF_LogWrite("User routine AtmBundleIntp finished", ESMF_LOGMSG_INFO) end subroutine AtmBundleIntp + !> Set scalar data from state for a particula name + subroutine State_SetScalar(value, scalar_id, State, mytask, scalar_name, scalar_count, rc) + real(ESMF_KIND_R8),intent(in) :: value + integer, intent(in) :: scalar_id + type(ESMF_State), intent(inout) :: State + integer, intent(in) :: mytask + character(len=*), intent(in) :: scalar_name + integer, intent(in) :: scalar_count + integer, intent(inout) :: rc !< return code + + ! local variables + type(ESMF_Field) :: field + real(ESMF_KIND_R8), pointer :: farrayptr(:,:) + character(len=*), parameter :: subname='(DATM_cap:State_SetScalar)' + !-------------------------------------------------------- + + rc = ESMF_SUCCESS + + call ESMF_StateGet(State, itemName=trim(scalar_name), field=field, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (mytask == 0) then + call ESMF_FieldGet(field, farrayPtr=farrayptr, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (scalar_id < 0 .or. scalar_id > scalar_count) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=subname//": ERROR in scalar_id", line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + + farrayptr(scalar_id,1) = value + endif + + end subroutine State_SetScalar end module AtmFieldUtils diff --git a/DATM/AtmInternalFields.F90 b/DATM/AtmInternalFields.F90 index a244f82..6c46f94 100644 --- a/DATM/AtmInternalFields.F90 +++ b/DATM/AtmInternalFields.F90 @@ -15,10 +15,14 @@ module AtmInternalFields !from model_configure integer, public :: iatm,jatm,nfhout + integer, public :: scalar_field_count = 0 + integer, public :: scalar_field_idx_grid_nx = 0 + integer, public :: scalar_field_idx_grid_ny = 0 real(kind=ESMF_KIND_R8), public :: dt_atmos character(len=ESMF_MAXSTR), public :: filename_base character(len=ESMF_MAXSTR), public :: cdate0 character(len=ESMF_MAXSTR), public :: dirpath = 'DATM_INPUT/' + character(len=ESMF_MAXSTR), public :: scalar_field_name = '' ! the forward and backward timestamps real(kind=ESMF_KIND_R8), public :: hfwd, hbak diff --git a/DATM/datm.F90 b/DATM/datm.F90 index c2d80ed..3af6dde 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -110,7 +110,9 @@ subroutine InitializeP0(model, importState, exportState, externalClock, rc) type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock integer, intent(out) :: rc + logical :: isPresent, isSet + integer :: iostat character(len=10) :: value character(len=ESMF_MAXSTR) :: msgString @@ -159,6 +161,54 @@ subroutine InitializeP0(model, importState, exportState, externalClock, rc) write(msgString,'(A,l6)')'DATM Profile_memory = ',profile_memory call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + + scalar_field_count = 0 + call NUOPC_CompAttributeGet(model, name="ScalarFieldCount", value=value, & + isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(value, '(i)', iostat=iostat) scalar_field_count + if (iostat /= 0) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="DATM : ScalarFieldCount not an integer: "//trim(value), & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + write(msgString,*) scalar_field_count + call ESMF_LogWrite('DATM: ScalarFieldCount = '//trim(msgString), ESMF_LOGMSG_INFO) + endif + + scalar_field_idx_grid_nx = 0 + call NUOPC_CompAttributeGet(model, name="ScalarFieldIdxGridNX", value=value, & + isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(value, '(i)', iostat=iostat) scalar_field_idx_grid_nx + if (iostat /= 0) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="DATM : ScalarFieldIdxGridNX not an integer: "//trim(value), & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + write(msgString,*) scalar_field_idx_grid_nx + call ESMF_LogWrite('DATM: ScalarFieldIdxGridNX = '//trim(msgString), ESMF_LOGMSG_INFO) + endif + + scalar_field_idx_grid_ny = 0 + call NUOPC_CompAttributeGet(model, name="ScalarFieldIdxGridNY", value=value, & + isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(value, '(i)', iostat=iostat) scalar_field_idx_grid_ny + if (iostat /= 0) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="DATM : ScalarFieldIdxGridNY not an integer: "//trim(value), & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + write(msgString,*) scalar_field_idx_grid_ny + call ESMF_LogWrite('DATM: ScalarFieldIdxGridNY = '//trim(msgString), ESMF_LOGMSG_INFO) + endif #endif call ESMF_LogWrite("User initialize routine InitP0 Atm finished", ESMF_LOGMSG_INFO) From 59e6b1ca4d82d95d95c86603a7ae7e309fc7cf16 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Wed, 13 May 2020 18:16:31 -0600 Subject: [PATCH 10/14] add the name --- DATM/datm.F90 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/DATM/datm.F90 b/DATM/datm.F90 index 3af6dde..78f9f6a 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -113,7 +113,7 @@ subroutine InitializeP0(model, importState, exportState, externalClock, rc) logical :: isPresent, isSet integer :: iostat - character(len=10) :: value + character(len=64) :: value character(len=ESMF_MAXSTR) :: msgString rc = ESMF_SUCCESS @@ -162,6 +162,15 @@ subroutine InitializeP0(model, importState, exportState, externalClock, rc) write(msgString,'(A,l6)')'DATM Profile_memory = ',profile_memory call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + scalar_field_name = "" + call NUOPC_CompAttributeGet(model, name="ScalarFieldName", value=value, & + isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + scalar_field_name = trim(value) + call ESMF_LogWrite('DATM: ScalarFieldName = '//trim(scalar_field_name), ESMF_LOGMSG_INFO) + endif + scalar_field_count = 0 call NUOPC_CompAttributeGet(model, name="ScalarFieldCount", value=value, & isPresent=isPresent, isSet=isSet, rc=rc) From f4a59ebda902f2f40ed9445bc988032f18e7a933 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Thu, 14 May 2020 07:58:16 -0600 Subject: [PATCH 11/14] more scalar fixes --- DATM/AtmFieldUtils.F90 | 83 +++++++++++++++++++++++++++--------------- DATM/datm.F90 | 28 ++++++++++++++ 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/DATM/AtmFieldUtils.F90 b/DATM/AtmFieldUtils.F90 index e59abc8..f77b9fe 100644 --- a/DATM/AtmFieldUtils.F90 +++ b/DATM/AtmFieldUtils.F90 @@ -16,6 +16,7 @@ module AtmFieldUtils public :: AtmFieldsAdvertise, AtmFieldsRealize public :: AtmFieldCheck public :: AtmFieldDump + public :: State_SetScalar ! called by AtmInit, AtmRun public :: AtmForceFwd2Bak, AtmBundleCheck @@ -57,6 +58,13 @@ subroutine AtmFieldsAdvertise(state, field_defs, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return enddo + ! advertise scalars for cmeps + if (len_trim(scalar_field_name) > 0) then + call NUOPC_Advertise(state, standardName=trim(scalar_field_name), name=trim(scalar_field_name), rc=rc) + call ESMF_LogWrite("Advertise Field "// trim(scalar_field_name), ESMF_LOGMSG_INFO) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + endif + end subroutine AtmFieldsAdvertise !----------------------------------------------------------------------------- @@ -69,26 +77,15 @@ subroutine AtmFieldsRealize(state, grid, field_defs, tag, rc) character(len=*), intent(in ) :: tag integer, intent(out ) :: rc - integer :: localPet - type(ESMF_VM) :: vm type(ESMF_ArraySpec) :: arrayspecR8 type(ESMF_Field) :: field type(ESMF_StaggerLoc) :: staggerloc character(len=ESMF_MAXSTR) :: msgString - integer :: mpicom - integer :: npet integer :: ii,nfields - logical :: connected rc = ESMF_SUCCESS call ESMF_LogWrite("User routine AtmFieldsRealize "//trim(tag)//" started", ESMF_LOGMSG_INFO) - call ESMF_VMGetCurrent(vm, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_VMGet(vm, petCount=npet, mpiCommunicator=mpicom, localPet=localPet, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - nfields=size(field_defs) !print *,'found nfields = ',nfields,' to realize ',field_defs%field_name @@ -105,32 +102,29 @@ subroutine AtmFieldsRealize(state, grid, field_defs, tag, rc) name=trim(field_defs(ii)%shortname), rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - call NUOPC_Realize(state, grid, rc=rc) + call NUOPC_Realize(state, field=field, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_FieldGet(field, farrayPtr=field_defs(ii)%farrayPtr, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + ! initialize field_defs(ii)%farrayPtr = 0.0 - !do j = lbound(field_defs(ii)%farrayPtr,2),ubound(field_defs(ii)%farrayPtr,2) - ! do i = lbound(field_defs(ii)%farrayPtr,1),ubound(field_defs(ii)%farrayPtr,1) - ! field_defs(ii)%farrayPtr(i,j) = 0.0 - ! enddo - !enddo - enddo - !--------------------------------- - ! set scalar data in export state - !--------------------------------- + msgString = trim(tag)//" Field "// trim(field_defs(ii)%shortname) // " is connected on root pe" + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + enddo + ! realize scalars for cmeps if (len_trim(scalar_field_name) > 0) then - call State_SetScalar(dble(iatm),scalar_field_idx_grid_nx, state, localPet, & - scalar_field_name, scalar_field_count, rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return + call SetScalarField(field, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call NUOPC_Realize(state, field=field, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return - call State_SetScalar(dble(jatm),scalar_field_idx_grid_ny, state, localPet, & - scalar_field_name, scalar_field_count, rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return + msgString = trim(tag)//" Field "// trim(scalar_field_name) // " is connected on root pe" + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) endif call ESMF_LogWrite("User routine AtmFieldsRealize "//trim(tag)//" finished", ESMF_LOGMSG_INFO) @@ -436,7 +430,9 @@ subroutine AtmBundleIntp(gcomp, exportState, externalClock, hour, rc) call ESMF_LogWrite("User routine AtmBundleIntp finished", ESMF_LOGMSG_INFO) end subroutine AtmBundleIntp - !> Set scalar data from state for a particula name + !----------------------------------------------------------------------------- + + !> Set scalar data from state for a particular name subroutine State_SetScalar(value, scalar_id, State, mytask, scalar_name, scalar_count, rc) real(ESMF_KIND_R8),intent(in) :: value integer, intent(in) :: scalar_id @@ -449,7 +445,7 @@ subroutine State_SetScalar(value, scalar_id, State, mytask, scalar_name, scalar_ ! local variables type(ESMF_Field) :: field real(ESMF_KIND_R8), pointer :: farrayptr(:,:) - character(len=*), parameter :: subname='(DATM_cap:State_SetScalar)' + character(len=*), parameter :: subname='(DATM: State_SetScalar)' !-------------------------------------------------------- rc = ESMF_SUCCESS @@ -471,4 +467,33 @@ subroutine State_SetScalar(value, scalar_id, State, mytask, scalar_name, scalar_ endif end subroutine State_SetScalar + + !----------------------------------------------------------------------------- + + subroutine SetScalarField(field, rc) + + ! create a field with scalar data on the root pe + type(ESMF_Field), intent(inout) :: field + integer, intent(inout) :: rc + + ! local variables + type(ESMF_Distgrid) :: distgrid + type(ESMF_Grid) :: grid + character(len=*), parameter :: subname='(DATM: SetScalarField)' + + rc = ESMF_SUCCESS + + ! create a DistGrid with a single index space element, which gets mapped onto DE 0. + distgrid = ESMF_DistGridCreate(minIndex=(/1/), maxIndex=(/1/), rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + grid = ESMF_GridCreate(distgrid, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + ! num of scalar values + field = ESMF_FieldCreate(name=trim(scalar_field_name), grid=grid, typekind=ESMF_TYPEKIND_R8, & + ungriddedLBound=(/1/), ungriddedUBound=(/scalar_field_count/), gridToFieldMap=(/2/), rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + end subroutine SetScalarField end module AtmFieldUtils diff --git a/DATM/datm.F90 b/DATM/datm.F90 index 78f9f6a..836031d 100644 --- a/DATM/datm.F90 +++ b/DATM/datm.F90 @@ -17,6 +17,7 @@ module DAtm use AtmFieldUtils, only : AtmFieldsAdvertise, AtmFieldsRealize use AtmFieldUtils, only : AtmFieldDump use AtmFieldUtils, only : AtmFieldCheck + use AtmFieldUtils, only : State_SetScalar use AtmGridUtils, only : WriteCoord, WriteMask ! AtmInit called by InitializeP2, AtmRun called by ModelAdvance @@ -330,6 +331,7 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) type(ESMF_State) :: exportState type(ESMF_Clock) :: externalClock type(ESMF_Time) :: currTime + type(ESMF_VM) :: vm integer, intent(out) :: rc ! local variables @@ -340,12 +342,19 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) character(ESMF_MAXSTR) :: timestr character(ESMF_MAXSTR) :: fname + integer :: localPet, npet, mpicom integer :: ii, nfields rc = ESMF_SUCCESS call ESMF_LogWrite("User initialize routine InitP2 Atm started", ESMF_LOGMSG_INFO) + call ESMF_VMGetCurrent(vm, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_VMGet(vm, petCount=npet, mpiCommunicator=mpicom, localPet=localPet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_ClockPrint(externalClock, options="currTime", & preString="InitP2 Atm CLOCK_EARTH current: ", & unit=msgString) @@ -408,6 +417,25 @@ subroutine InitializeP2(model, importState, exportState, externalClock, rc) call ESMF_LogWrite(trim(AtmBundleFields(ii)%shortname)//' set to Updated', ESMF_LOGMSG_INFO) enddo !ii + + ! set scalars for cmeps + if(len_trim(scalar_field_name) > 0) then + call State_SetScalar(dble(iatm),scalar_field_idx_grid_nx, exportState, localPet, & + scalar_field_name, scalar_field_count, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call State_SetScalar(dble(jatm),scalar_field_idx_grid_ny, exportState, localPet, & + scalar_field_name, scalar_field_count, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_StateGet(exportState, itemName=trim(scalar_field_name), field=field, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call NUOPC_SetAttribute(field, name="Updated", value="true", rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_LogWrite(trim(scalar_field_name)//' set to Updated', ESMF_LOGMSG_INFO) + end if ! the component needs to indicate that it is fully done with ! initializing its data: From b48e8391b69c83265417814c7ebc8d6c1dbd8669 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Thu, 4 Jun 2020 11:33:28 +0000 Subject: [PATCH 12/14] add abort of datm forcing file is not found --- DATM/AtmForce.F90 | 77 ++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/DATM/AtmForce.F90 b/DATM/AtmForce.F90 index 214d6b1..d6737f3 100644 --- a/DATM/AtmForce.F90 +++ b/DATM/AtmForce.F90 @@ -19,11 +19,15 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) type(ESMF_Time) :: currTime type(ESMF_Time) :: nextTime type(ESMF_TimeInterval) :: timeStep + type(ESMF_VM) :: vm integer(kind=ESMF_KIND_I4) :: year, month, day, hour, jday + integer :: mytask integer :: ii,nfields integer :: iii, iid, iiu + logical :: fexists + integer :: fpresent(1) character(len=ESMF_MAXSTR) :: varname character(len=ESMF_MAXSTR) :: filename @@ -45,6 +49,12 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) call ESMF_LogWrite("User routine AtmForce started", ESMF_LOGMSG_INFO) + call ESMF_GridCompGet(gcomp, vm=vm, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_VMGet(vm, localPet=mytask, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + ! at initialization, get the current forecast hour file, not the forward ! forecast hour if(initmode .eq. 0)then @@ -76,35 +86,48 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) call ESMF_TimeGet(nextTime,h_r8=hfwd,rc=rc) endif - write(msgString,'(3a,f12.3)')'using ',trim(filename),' at fwd clock hour ',real(hfwd,4) + write(msgString,'(3a,f12.3)')'using ',trim(filename),' at fwd clock hour ',hfwd call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) - ! read the Atm field data into the Fwd bundle - nfields = size(AtmBundleFields) - do ii = 1,nfields - if(AtmBundleFields(ii)%isPresent)then - varname = trim(AtmBundleFields(ii)%file_varname) - - ! get the '_fwd' field - call ESMF_FieldBundleGet(AtmBundleFwd, & - fieldName=trim(AtmBundleFields(ii)%field_name)//'_fwd', & - field=field, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_FieldRead(field, & - fileName=trim(filename), & - variableName = trim(varname), & - rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_FieldGet(field,farrayPtr=AtmBundleFields(ii)%farrayPtr_fwd,rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - - write(msgString,'(i6,2a18,f14.5)')ii,' inside AtmForce ',trim(varname), & - AtmBundleFields(ii)%farrayPtr_fwd(iprnt,jprnt) - call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) - endif !isPresent - enddo + fexists = .false. + if(mytask == 0)inquire(file=trim(filename), exist=fexists) + if(fexists) fpresent(1) = 1 + call ESMF_VMBroadCast(vm, fpresent, count=1, rootPet=0, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if(fpresent(1) == 1)fexists = .true. + + if(.not. fexists)then + if(mytask == 0)print *,'Cannot read file '//trim(filename)//': Aborting' + call ESMF_LogWrite('Cannot read file '//trim(filename)//': Aborting', ESMF_LOGMSG_INFO) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + else + ! read the Atm field data into the Fwd bundle + nfields = size(AtmBundleFields) + do ii = 1,nfields + if(AtmBundleFields(ii)%isPresent)then + varname = trim(AtmBundleFields(ii)%file_varname) + + ! get the '_fwd' field + call ESMF_FieldBundleGet(AtmBundleFwd, & + fieldName=trim(AtmBundleFields(ii)%field_name)//'_fwd', & + field=field, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_FieldRead(field, & + fileName=trim(filename), & + variableName = trim(varname), & + rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_FieldGet(field,farrayPtr=AtmBundleFields(ii)%farrayPtr_fwd,rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + write(msgString,'(i6,2a18,f14.5)')ii,' inside AtmForce ',trim(varname), & + AtmBundleFields(ii)%farrayPtr_fwd(iprnt,jprnt) + call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO) + endif !isPresent + enddo + end if ! Check for fields which are not Present but are needed ! Not very clean---would have been better to create this field in the forcing file From 383553b1113982eb8b0ee737ad6964139160abd9 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Thu, 4 Jun 2020 08:26:58 -0500 Subject: [PATCH 13/14] add default value for dbug_flag --- DATM/AtmInternalFields.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATM/AtmInternalFields.F90 b/DATM/AtmInternalFields.F90 index a244f82..c447304 100644 --- a/DATM/AtmInternalFields.F90 +++ b/DATM/AtmInternalFields.F90 @@ -59,7 +59,8 @@ module AtmInternalFields type(AtmField_Definition), public :: AtmBundleFields(AtmFieldCount) - integer, public :: lPet, petCnt, dbug_flag + integer, public :: lPet, petCnt + integer, public :: dbug_flag = 0 ! a diagnostic point to print at integer, public :: iprnt, jprnt integer :: icnt From 76a49ed1483d0b91027998431107eb91348f0445 Mon Sep 17 00:00:00 2001 From: "denise.worthen" Date: Thu, 4 Jun 2020 18:03:18 +0000 Subject: [PATCH 14/14] Add abort if reading the data file does not return success Initialize fpresent --- DATM/AtmForce.F90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DATM/AtmForce.F90 b/DATM/AtmForce.F90 index d6737f3..940bf3b 100644 --- a/DATM/AtmForce.F90 +++ b/DATM/AtmForce.F90 @@ -27,7 +27,7 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) integer :: ii,nfields integer :: iii, iid, iiu logical :: fexists - integer :: fpresent(1) + integer :: fpresent(1) = 0 character(len=ESMF_MAXSTR) :: varname character(len=ESMF_MAXSTR) :: filename @@ -117,6 +117,10 @@ subroutine AtmForce(gcomp,exportState,externalClock,initmode,rc) fileName=trim(filename), & variableName = trim(varname), & rc=rc) + if( rc /= ESMF_SUCCESS)then + call ESMF_LogWrite('Problem reading '//trim(filename), ESMF_LOGMSG_INFO) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_FieldGet(field,farrayPtr=AtmBundleFields(ii)%farrayPtr_fwd,rc=rc)