merge to latest develop

.gitignore

@@ -1,3 +1,4 @@
 *.o
 *.mod
 *.a
+*.pyc

CMakeLists.txt

@@ -0,0 +1,130 @@
+if( NOT DEFINED PHYS)
+  set(PHYS gfs)
+endif()
+
+if (PHYS MATCHES "nam")
+  add_definitions(-DNAM_phys)
+endif()
+
+if(CCPP)
+  add_definitions(-DCCPP)
+  if(STATIC)
+    add_definitions(-DSTATIC)
+  endif()
+endif()
+
+add_subdirectory(cpl)
+add_subdirectory(${PHYS}physics)
+add_subdirectory(ipd)
+add_subdirectory(io)
+
+# fv3core library
+
+if(NOT CCPP)
+set(FAST_PHYSICS_SRCS_F90 atmos_cubed_sphere/model/fv_cmp.F90)
+endif()
+
+add_library(
+    fv3core
+
+    atmos_cubed_sphere/model/a2b_edge.F90
+    atmos_cubed_sphere/model/multi_gases.F90
+    atmos_cubed_sphere/model/boundary.F90
+    atmos_cubed_sphere/model/dyn_core.F90
+    atmos_cubed_sphere/model/fv_arrays.F90
+    ${FAST_PHYSICS_SRCS_F90}
+    atmos_cubed_sphere/model/fv_control.F90
+    atmos_cubed_sphere/model/fv_dynamics.F90
+    atmos_cubed_sphere/model/fv_fill.F90
+    atmos_cubed_sphere/model/fv_grid_utils.F90
+    atmos_cubed_sphere/model/fv_mapz.F90
+    atmos_cubed_sphere/model/fv_nesting.F90
+    atmos_cubed_sphere/model/fv_regional_bc.F90
+    atmos_cubed_sphere/model/fv_sg.F90
+    atmos_cubed_sphere/model/fv_tracer2d.F90
+    atmos_cubed_sphere/model/fv_update_phys.F90
+    atmos_cubed_sphere/model/sw_core.F90
+    atmos_cubed_sphere/model/tp_core.F90
+    atmos_cubed_sphere/model/nh_core.F90
+    atmos_cubed_sphere/model/nh_utils.F90
+    atmos_cubed_sphere/tools/external_ic.F90
+    atmos_cubed_sphere/tools/external_sst.F90
+    atmos_cubed_sphere/tools/fv_diagnostics.F90
+    atmos_cubed_sphere/tools/fv_eta.F90
+    atmos_cubed_sphere/tools/fv_grid_tools.F90
+    atmos_cubed_sphere/tools/fv_io.F90
+    atmos_cubed_sphere/tools/fv_mp_mod.F90
+    atmos_cubed_sphere/tools/fv_nudge.F90
+    atmos_cubed_sphere/tools/fv_treat_da_inc.F90
+    atmos_cubed_sphere/tools/fv_iau_mod.F90
+    atmos_cubed_sphere/tools/fv_restart.F90
+    atmos_cubed_sphere/tools/fv_surf_map.F90
+    atmos_cubed_sphere/tools/fv_timing.F90
+    atmos_cubed_sphere//tools/init_hydro.F90
+    atmos_cubed_sphere/tools/sim_nc_mod.F90
+    atmos_cubed_sphere/tools/sorted_index.F90
+    atmos_cubed_sphere/tools/test_cases.F90
+    atmos_cubed_sphere/driver/fvGFS/DYCORE_typedefs.F90
+    atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90
+    atmos_cubed_sphere/driver/fvGFS/atmosphere.F90
+)
+
+set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
+set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90  APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
+
+target_link_libraries(fv3core PRIVATE fms)
+target_link_libraries(fv3core PRIVATE ${PHYS}physics)
+target_link_libraries(fv3core PRIVATE ipd)
+
+target_include_directories(fv3core PRIVATE ${ESMF_MOD})
+
+# end of fv3core
+
+###############################################################################
+### ccpp
+###############################################################################
+if(CCPP)
+  add_subdirectory(ccpp)
+  add_subdirectory(ccpp/driver)
+  add_dependencies(${PHYS}physics ccpp ccppphys)
+  add_dependencies(ccppdriver ccpp ccppphys)
+  add_dependencies(ccppphys ccpp)
+
+  target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
+  target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/driver)
+endif()
+
+set_target_properties(fv3core PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(fv3core PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)
+
+# When specifying an out-of-tree source a binary directory must be explicitly specified.
+#add_subdirectory(../stochastic_physics stochastic_physics)
+add_subdirectory(stochastic_physics)
+
+add_library(
+    fv3cap
+
+    atmos_model.F90
+    fv3_cap.F90
+    module_fv3_config.F90
+    module_fcst_grid_comp.F90
+    time_utils.F90
+)
+
+if(CCPP)
+target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod)
+endif()
+target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics)
+target_compile_definitions(fv3cap PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
+target_include_directories(fv3cap PRIVATE ${ESMF_MOD})
+
+target_link_libraries(fv3cap PRIVATE fms)
+target_link_libraries(fv3cap PRIVATE fv3cpl)
+target_link_libraries(fv3cap PRIVATE ${PHYS}physics)
+target_link_libraries(fv3cap PRIVATE ipd)
+target_link_libraries(fv3cap PRIVATE io)
+target_link_libraries(fv3cap PRIVATE fv3core)
+target_link_libraries(fv3cap PRIVATE stochastic_physics)
+
+set_target_properties(fv3cap PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(fv3cap PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)

atmos_model.F90

@@ -113,7 +113,8 @@ module atmos_model_mod
                               FV3GFS_diag_register, FV3GFS_diag_output,  &
                               DIAG_SIZE
 use fv_iau_mod,         only: iau_external_data_type,getiauforcing,iau_initialize
-use module_fv3_config,  only: output_1st_tstep_rst, first_kdt, nsout
+use module_fv3_config,  only: output_1st_tstep_rst, first_kdt, nsout,    &
+                              frestart, restart_endfcst
 
 !-----------------------------------------------------------------------
 
@@ -221,7 +222,8 @@ module atmos_model_mod
   logical,parameter :: flip_vc = .true.
 #endif
 
-  real(kind=IPD_kind_phys), parameter :: zero=0.0, one=1.0
+  real(kind=IPD_kind_phys), parameter :: zero = 0.0_IPD_kind_phys, &
+                                         one  = 1.0_IPD_kind_phys
 
 contains
 
@@ -944,17 +946,19 @@ end subroutine update_atmos_model_state
 subroutine atmos_model_end (Atmos)
   type (atmos_data_type), intent(inout) :: Atmos
 !---local variables
-  integer :: idx
+  integer :: idx, seconds
 #ifdef CCPP
   integer :: ierr
 #endif
 
 !-----------------------------------------------------------------------
 !---- termination routine for atmospheric model ----
 
-    call atmosphere_end (Atmos % Time, Atmos%grid)
-    call FV3GFS_restart_write (IPD_Data, IPD_Restart, Atm_block, &
-                               IPD_Control, Atmos%domain)
+    call atmosphere_end (Atmos % Time, Atmos%grid, restart_endfcst)
+    if(restart_endfcst) then
+      call FV3GFS_restart_write (IPD_Data, IPD_Restart, Atm_block, &
+                                 IPD_Control, Atmos%domain)
+    endif
 
 #ifdef CCPP
 !   Fast physics (from dynamics) are finalized in atmosphere_end above;
@@ -1457,6 +1461,24 @@ subroutine update_atmos_chemistry(state, rc)
         enddo
       enddo
 
+      ! -- zero out accumulated fields
+!$OMP parallel do default (none) &
+!$OMP             shared  (nj, ni, Atm_block, IPD_Control, IPD_Data) &
+!$OMP             private (j, jb, i, ib, nb, ix)
+      do j = 1, nj
+        jb = j + Atm_block%jsc - 1
+        do i = 1, ni
+          ib = i + Atm_block%isc - 1
+          nb = Atm_block%blkno(ib,jb)
+          ix = Atm_block%ixp(ib,jb)
+          IPD_Data(nb)%coupling%rainc_cpl(ix)  = zero
+          if (.not.IPD_Control%cplflx) then
+            IPD_Data(nb)%coupling%rain_cpl(ix) = zero
+            IPD_Data(nb)%coupling%snow_cpl(ix) = zero
+          end if
+        enddo
+      enddo
+
       if (IPD_Control%debug) then
         ! -- diagnostics
         write(6,'("update_atmos: prsi   - min/max/avg",3g16.6)') minval(prsi),   maxval(prsi),   sum(prsi)/size(prsi)

ccpp/driver/CMakeLists.txt

@@ -0,0 +1,27 @@
+
+if(NOT DYN32)
+remove_definitions(-DOVERLOAD_R8)
+remove_definitions(-DOVERLOAD_R4)
+endif()
+
+message ("Force 64 bits in CCPP_layer")
+if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
+    string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
+elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8")
+endif()
+
+add_library(
+    ccppdriver
+
+    CCPP_driver.F90
+)
+
+target_link_libraries(ccppdriver ${PHYS}physics)
+target_link_libraries(ccppdriver ccpp)
+target_link_libraries(ccppdriver ccppphys)
+
+target_include_directories(ccppdriver PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
+
+set_target_properties(ccppdriver PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(ccppdriver PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)

cpl/CMakeLists.txt

@@ -0,0 +1,12 @@
+
+add_library(
+    fv3cpl
+
+    module_cplfields.F90
+    module_cap_cpl.F90
+)
+
+target_include_directories(fv3cpl PRIVATE ${ESMF_MOD})
+
+set_target_properties(fv3cpl PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(fv3cpl PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)

fv3_cap.F90

@@ -30,7 +30,7 @@ module fv3gfs_cap_mod
                                     calendar, calendar_type, cpl,            &
                                     force_date_from_configure,               &
                                     cplprint_flag,output_1st_tstep_rst,      &
-                                    first_kdt                            
+                                    first_kdt,num_restart_interval       
 
   use module_fv3_io_def,      only: num_pes_fcst,write_groups,app_domain,    &
                                     num_files, filename_base,                &
@@ -278,9 +278,16 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     CALL ESMF_ConfigLoadFile(config=CF ,filename='model_configure' ,rc=RC)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
 !
-    CALL ESMF_ConfigGetAttribute(config=CF,value=restart_interval, &
-                                 label ='restart_interval:',rc=rc)
+    num_restart_interval = ESMF_ConfigGetLen(config=CF, label ='restart_interval:',rc=rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+    if(mype == 0) print *,'af nems config,num_restart_interval=',num_restart_interval
+    if (num_restart_interval<=0) num_restart_interval = 1
+    allocate(restart_interval(num_restart_interval))
+    restart_interval = 0
+    CALL  ESMF_ConfigGetAttribute(CF,valueList=restart_interval,label='restart_interval:', &
+      count=num_restart_interval, rc=RC)
+    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+    if(mype == 0) print *,'af nems config,restart_interval=',restart_interval
 !
     CALL ESMF_ConfigGetAttribute(config=CF,value=calendar, &
                                  label ='calendar:',rc=rc)
@@ -326,9 +333,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
                                    label ='app_domain:',rc=rc)
       if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
 
-      if(mype == 0) print *,'af nems config,restart_interval=',restart_interval, &
-      'quilting=',quilting,'write_groups=',write_groups,wrttasks_per_group,      &
-      'calendar=',trim(calendar),'calendar_type=',calendar_type
+      if(mype == 0) print *,'af nems config,quilting=',quilting,'write_groups=', &
+        write_groups,wrttasks_per_group,'calendar=',trim(calendar),'calendar_type=',calendar_type
 !
       CALL ESMF_ConfigGetAttribute(config=CF,value=num_files, &
                                    label ='num_files:',rc=rc)