diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..f0167a035 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile.* +run_test.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..a364b3d4d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,44 @@ +language: C + +sudo: false + +services: + - docker + +env: + global: + CACHE_IMAGE: ofuhrer/fv3atm_test + +before_install: + - test -n $CC && unset CC + +jobs: + include: + - stage: build docker images + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - cd test + - docker pull $CACHE_IMAGE:base || true + - docker build + --target base + --cache-from $CACHE_IMAGE:base + --tag $CACHE_IMAGE:base + --file Dockerfile.base + "." + - docker push $CACHE_IMAGE:base + - cd ../ + - docker pull $CACHE_IMAGE:test || true + - docker build + --target test + --cache-from $CACHE_IMAGE:test + --tag $CACHE_IMAGE:test + --file test/Dockerfile.test + "." + - docker push $CACHE_IMAGE:test + - stage: run tests + script: + docker run + -it + --rm + --mount type=bind,source=$TRAVIS_BUILD_DIR/test/work,target=/work + --name=fv3_atm ofuhrer/fv3atm_test:test diff --git a/test/.dockerignore b/test/.dockerignore new file mode 100644 index 000000000..1d8b3c204 --- /dev/null +++ b/test/.dockerignore @@ -0,0 +1 @@ +Dockerfile.* diff --git a/test/Dockerfile.base b/test/Dockerfile.base new file mode 100644 index 000000000..9c9815672 --- /dev/null +++ b/test/Dockerfile.base @@ -0,0 +1,124 @@ +## --------------------------------------------------------------------------------- +## Build environment image +FROM ubuntu:19.10 AS base + +MAINTAINER Oliver Fuhrer, oliverf@vulcan.com + +ENV ESMF_BRANCH=ESMF_8_0_0 \ + NCEP_SHA=3da51e139d5cd731c9fc27f39d88cb4e1328212b \ + FMS_SHA=f68878549364e39591216874d5a922c06b7d3a91 \ + STOCHASTIC_SHA=01739da377d603c9c8d6d666afe2442260fd79d1 \ + FV3ATM_SHA=b48b640f4d8cae978e2d98e149412054d74dcce7 \ + FV3CONFIG_BRANCH=v0.3.2 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \ + curl \ + wget \ + gcc \ + git \ + libblas-dev \ + liblapack-dev \ + libnetcdf-dev \ + libnetcdff-dev \ + perl \ + make \ + rsync \ + libffi-dev \ + openssl \ + libopenmpi3 \ + python3 \ + python3-pip \ + bats + +RUN ln -s /bin/python3 /bin/python && \ + ln -s /bin/pip3 /bin/pip && \ + pip install --no-cache-dir pyyaml + +RUN git config --global user.email "you@example.com" && \ + git config --global user.name "Your Name" + +## --------------------------------------------------------------------------------- +## Build NCEPlibs +RUN git config --global http.sslverify false && \ + git clone https://github.com/NCAR/NCEPlibs.git /NCEPlibs && \ + mkdir /opt/NCEPlibs && \ + cd NCEPlibs && \ + git checkout $NCEP_SHA && \ + echo "y" | ./make_ncep_libs.sh -s linux -c gnu -d /opt/NCEPlibs -o 1 && \ + /bin/rm -rf /NCEPlibs && mv /opt/NCEPlibs /NCEPlibs + +## --------------------------------------------------------------------------------- +## Build ESMF +ENV ESMF_DIR=/esmf \ + ESMF_INSTALL_PREFIX=/usr/local/esmf \ + ESMF_NETCDF_INCLUDE=/usr/include \ + ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" \ + ESMF_BOPT=O3 +RUN git clone -b $ESMF_BRANCH --depth 1 https://git.code.sf.net/p/esmf/esmf $ESMF_DIR && \ + cd $ESMF_DIR && \ + make lib -j8 && \ + make install && \ + make installcheck + +## --------------------------------------------------------------------------------- +## Build FMS +COPY patches/PATCH-FMS-affinity.c-fix-getpid.patch / +COPY patches/PATCH-FMS-libFMS_Makefile.am-add-sat-vapor-pres.patch / +RUN git clone https://github.com/NOAA-GFDL/FMS.git /FMS && \ + cd /FMS && git checkout $FMS_SHA && \ + git am /PATCH-FMS-affinity.c-fix-getpid.patch && \ + git am /PATCH-FMS-libFMS_Makefile.am-add-sat-vapor-pres.patch && \ + CC=mpicc FC=mpifort LDFLAGS="-L/usr/lib" LOG_DRIVER_FLAGS="--comments" \ + CPPFLAGS="-I/usr/include -Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS" \ + FCFLAGS="-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp" \ + autoreconf --install && \ + CC=mpicc FC=mpifort LDFLAGS="-L/usr/lib" LOG_DRIVER_FLAGS="--comments" \ + CPPFLAGS="-I/usr/include -Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS" \ + FCFLAGS="-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp" \ + ./configure && \ + CC=mpicc FC=mpifort LDFLAGS="-L/usr/lib" LOG_DRIVER_FLAGS="--comments" \ + CPPFLAGS="-I/usr/include -Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS" \ + FCFLAGS="-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp" \ + make -j8 && \ + mv /FMS/*/*.mod /FMS/*/*.o /FMS/*/*.h /FMS/ + +## --------------------------------------------------------------------------------- +## Build fv3atm executable + +ENV NCEP_DIR=/NCEPlibs \ + FMS_DIR=/FMS \ + ESMF_DIR=/usr/local/esmf + +ENV ESMF_INC="-I/usr/local/esmf/include -I${ESMF_DIR}/mod/modO3/Linux.gfortran.64.mpiuni.default/" \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/:${FMS_DIR}/libFMS/.libs/ + +COPY patches/PATCH-stochastic_physics-update_ca.F90-fix-logical-comparisons.patch / +RUN git clone https://github.com/noaa-psd/stochastic_physics.git /stochastic_physics && \ + cd /stochastic_physics && git checkout $STOCHASTIC_SHA + +COPY patches/PATCH-fv3atm-apply-fixes.patch / +RUN git clone https://github.com/NOAA-EMC/fv3atm.git /FV3 && \ + cd /FV3 && git checkout $FV3ATM_SHA && \ + git submodule update --init && \ + git am /PATCH-fv3atm-apply-fixes.patch && \ + mkdir namphysics && mkdir conf + +RUN /bin/rm -f /PATCH-* +COPY etc/make.rules /FV3/conf/make.rules +COPY etc/configure.fv3.gnu_docker /FV3/conf/configure.fv3 + +# compile base (should speedup compilation of injected code) +#RUN cd /FV3 && make clean && make libs -j8 +#RUN cd /FV3/atmos_cubed_sphere && make clean && cd /FV3 && make -j8 + +## ## --------------------------------------------------------------------------------- +## ## Install fv3config +## RUN git clone -b $FV3CONFIG_BRANCH https://github.com/VulcanClimateModeling/fv3config.git && \ +## cd fv3config && \ +## pip install --no-cache-dir -e . && \ +## python -m fv3config.download_data + +## --------------------------------------------------------------------------------- +## Not meant for entering +CMD ["bash"] diff --git a/test/Dockerfile.test b/test/Dockerfile.test new file mode 100644 index 000000000..24946dc04 --- /dev/null +++ b/test/Dockerfile.test @@ -0,0 +1,25 @@ +## --------------------------------------------------------------------------------- +## Build test image +FROM ofuhrer/fv3atm_test:base AS test + +MAINTAINER Oliver Fuhrer, oliverf@vulcan.com + +# inject code into container image +#RUN /bin/rm -rf /FV3/atmos_cubed_sphere && \ +# mkdir /FV3/atmos_cubed_sphere +#COPY GFDL_tools /FV3/atmos_cubed_sphere/GFDL_tools +#COPY driver /FV3/atmos_cubed_sphere/driver +#COPY model /FV3/atmos_cubed_sphere/model +#COPY tools /FV3/atmos_cubed_sphere/tools +#COPY makefile /FV3/atmos_cubed_sphere/makefile + +# rebuild (with injected code) +RUN cd /FV3 && make libs -j8 && make -j8 + +# setup everything for testing +ENV LD_LIBRARY_PATH=/FMS/libFMS/.libs:/usr/local/esmf/lib/libO3/Linux.gfortran.64.mpiuni.default + +RUN mkdir /work +WORKDIR /work + +CMD ["/work/run_test.sh"] diff --git a/test/etc/configure.fv3.gnu_docker b/test/etc/configure.fv3.gnu_docker new file mode 100644 index 000000000..f92810d14 --- /dev/null +++ b/test/etc/configure.fv3.gnu_docker @@ -0,0 +1,139 @@ + +############ +# commands # +############ +FC = FC=gfortran mpifort +CC = CC=gcc mpicc +CXX = g++ +LD = mpifort + +INCLUDE = + +######### +# flags # +######### +# default is 64-bit OpenMP non-hydrostatic build +DEBUG = +REPRO = Y +VERBOSE = +OPENMP = Y +AVX2 = N +HYDRO = N +32BIT = N + + +NCEPLIBS_INC = /NCEPlibs/include +INCLUDE += -I$(NCEPLIBS_INC) + +NCEPLIBS_DIR = /NCEPlibs/lib +NEMSIO_LIB = $(NCEPLIBS_DIR)/libnemsio_d.a +BACIO_LIB4 = $(NCEPLIBS_DIR)/libbacio_4.a +SP_LIBd = $(NCEPLIBS_DIR)/libsp_v2.0.2_d.a +W3EMC_LIBd = $(NCEPLIBS_DIR)/libw3emc_d.a +W3NCO_LIBd = $(NCEPLIBS_DIR)/libw3nco_d.a +NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) + +############################################## +# Need to use at least GNU Make version 3.81 # +############################################## +need := 3.81 +ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) +ifneq ($(need),$(ok)) +$(error Need at least make version $(need). Load module gmake/3.81) +endif + +NETCDF_DIR = /usr +NETCDF_ROOT = $(NETCDF_DIR) +INCLUDE += -I$(NETCDF_ROOT)/include + +FPPFLAGS := -cpp -Wp,-w $(INCLUDE) -fPIC +CFLAGS := $(INCLUDE) -fPIC + +FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fPIC + +CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -DNO_INLINE_POST +CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML + +ifeq ($(HYDRO),Y) +CPPDEFS += +else +CPPDEFS += -DMOIST_CAPPA -DUSE_COND +endif + +ifeq ($(32BIT),Y) +CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 +else +FFLAGS += -fdefault-double-8 -fdefault-real-8 +endif + +ifeq ($(AVX2),Y) +FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +endif + +FFLAGS_OPT = -O2 +FFLAGS_REPRO = -O2 -g -fbacktrace +FFLAGS_DEBUG = -O0 -g -fbacktrace -fno-fast-math -ffree-line-length-none -fno-backslash -pedantic -Waliasing -Wampersand -Wline-truncation -Wsurprising -Wtabs -Wunderflow -fdump-core -ffpe-trap=invalid,zero,overflow -fbounds-check -finit-real=nan -finit-integer=9999999 -finit-logical=true -finit-character=35 + +TRANSCENDENTALS := -fast-transcendentals +FFLAGS_OPENMP = -fopenmp +FFLAGS_VERBOSE = -v -V -what + +CFLAGS += -D__IFC + +CFLAGS_OPT = -O2 +CFLAGS_REPRO = -O2 +CFLAGS_OPENMP = -fopenmp +CFLAGS_DEBUG = -O0 -g + +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits +CFLAGS_TEST = -O2 + +LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ +LDFLAGS_OPENMP := -fopenmp +LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M + +# start with blank LIBS +LIBS := + +LIBS += -lgfortran + +ifneq ($(REPRO),) +CFLAGS += $(CFLAGS_REPRO) +FFLAGS += $(FFLAGS_REPRO) +FAST := +else ifneq ($(DEBUG),) +CFLAGS += $(CFLAGS_DEBUG) +FFLAGS += $(FFLAGS_DEBUG) +FAST := +else ifneq ($(TEST),) +CFLAGS += $(CFLAGS_TEST) +FFLAGS += $(FFLAGS_TEST) +FAST := +else +CFLAGS += $(CFLAGS_OPT) +FFLAGS += $(FFLAGS_OPT) +FAST := $(TRANSCENDENTALS) +endif + +ifneq ($(OPENMP),) +CFLAGS += $(CFLAGS_OPENMP) +FFLAGS += $(FFLAGS_OPENMP) +LDFLAGS += $(LDFLAGS_OPENMP) +# to correct a loader bug on gaea: envars below set by module load intel +#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt +#LIBS += -lifcoremt +endif + +ifneq ($(VERBOSE),) +CFLAGS += $(CFLAGS_VERBOSE) +FFLAGS += $(FFLAGS_VERBOSE) +LDFLAGS += $(LDFLAGS_VERBOSE) +endif + +LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt + +LDFLAGS += $(LIBS) + diff --git a/test/etc/make.rules b/test/etc/make.rules new file mode 100644 index 000000000..d01397bf4 --- /dev/null +++ b/test/etc/make.rules @@ -0,0 +1,30 @@ +.SUFFIXES: +.SUFFIXES: .F90 .f90 .F .f .o .c + +.F90.f90: + $(CPP) $(CPPFLAGS) $< > $*.f90 + +.F.f: + $(CPP) $(CPPFLAGS) $< > $*.f + +.f.o: + $(FC) $(FFLAGS) $(OTHER_FFLAGS) -c $< -o $@ + +.f90.o: + $(FC) $(FFLAGS) $(OTHER_FFLAGS) -c $< -o $@ + +.F.o: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) $(OTHER_FFLAGS) -c $< -o $@ + +.F90.o: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) $(OTHER_FFLAGS) -c $< -o $@ + +.c.o: + $(CC) $(CPPDEFS) $(CPPFLAGS) $(CFLAGS) $(OTHERFLAGS) $(OTHER_CFLAGS) -c $< -o $@ + +depend: $(DEPEND_FILES) makefile + @echo "Building dependencies ..." + @ls -1 $(DEPEND_FILES) > Srcfiles + @echo "." > Filepath + @$(MKDEPENDS) -m Filepath Srcfiles > depend + @$(RM) -f Filepath Srcfiles diff --git a/test/patches/PATCH-FMS-affinity.c-fix-getpid.patch b/test/patches/PATCH-FMS-affinity.c-fix-getpid.patch new file mode 100644 index 000000000..5aaf2d312 --- /dev/null +++ b/test/patches/PATCH-FMS-affinity.c-fix-getpid.patch @@ -0,0 +1,65 @@ +From a4e586df827e148c3722ca507283a7ef9bba8b4d Mon Sep 17 00:00:00 2001 +From: Oliver Fuhrer +Date: Fri, 8 May 2020 00:19:02 -0700 +Subject: [PATCH] fix affinity.c to use syscall + +--- + affinity/affinity.c | 19 ++++++++++--------- + 1 file changed, 10 insertions(+), 9 deletions(-) + +diff --git a/affinity/affinity.c b/affinity/affinity.c +index cfd5285..bbaa5ee 100644 +--- a/affinity/affinity.c ++++ b/affinity/affinity.c +@@ -27,12 +27,13 @@ + #include + #include + +-#ifndef __APPLE__ +-static pid_t gettid(void) ++ ++static pid_t gettid1(void) + { +- return syscall(__NR_gettid); +-} ++#ifndef __APPLE__ ++ return syscall(SYS_gettid); + #endif ++} + + /* + * Returns this thread's CPU affinity, if bound to a single core, +@@ -44,8 +45,8 @@ int get_cpu_affinity(void) + cpu_set_t coremask; /* core affinity mask */ + + CPU_ZERO(&coremask); +- if (sched_getaffinity(gettid(),sizeof(cpu_set_t),&coremask) != 0) { +- fprintf(stderr,"Unable to get thread %d affinity. %s\n",gettid(),strerror(errno)); ++ if (sched_getaffinity(gettid1(),sizeof(cpu_set_t),&coremask) != 0) { ++ fprintf(stderr,"Unable to get thread %d affinity. %s\n",gettid1(),strerror(errno)); + } + + int cpu; +@@ -71,8 +72,8 @@ int get_cpuset(int fsz, int *output, int pe, _Bool debug) + cpu_set_t coremask; /* core affinity mask */ + + CPU_ZERO(&coremask); +- if (sched_getaffinity(gettid(),sizeof(cpu_set_t),&coremask) != 0) { +- fprintf(stderr,"Unable to get thread %d affinity. %s\n",gettid(),strerror(errno)); ++ if (sched_getaffinity(gettid1(),sizeof(cpu_set_t),&coremask) != 0) { ++ fprintf(stderr,"Unable to get thread %d affinity. %s\n",gettid1(),strerror(errno)); + } + + int cpu; +@@ -115,7 +116,7 @@ int set_cpu_affinity(int cpu) + + CPU_ZERO(&coremask); + CPU_SET(cpu,&coremask); +- if (sched_setaffinity(gettid(),sizeof(cpu_set_t),&coremask) != 0) { ++ if (sched_setaffinity(gettid1(),sizeof(cpu_set_t),&coremask) != 0) { + return -1; + } + #endif +-- +2.24.0 + diff --git a/test/patches/PATCH-FMS-libFMS_Makefile.am-add-sat-vapor-pres.patch b/test/patches/PATCH-FMS-libFMS_Makefile.am-add-sat-vapor-pres.patch new file mode 100644 index 000000000..e8e04007c --- /dev/null +++ b/test/patches/PATCH-FMS-libFMS_Makefile.am-add-sat-vapor-pres.patch @@ -0,0 +1,25 @@ +From 9631ca0bc102f351fa07f2d0a88f963f7d4e7441 Mon Sep 17 00:00:00 2001 +From: Oliver Fuhrer +Date: Fri, 8 May 2020 12:38:03 -0700 +Subject: [PATCH] add sat_vapor_pres to FMS + +--- + libFMS/Makefile.am | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/libFMS/Makefile.am b/libFMS/Makefile.am +index 68d237c..6aefef4 100644 +--- a/libFMS/Makefile.am ++++ b/libFMS/Makefile.am +@@ -87,6 +87,8 @@ libFMS_la_LIBADD += ${top_builddir}/exchange/libxgrid.la + libFMS_la_LIBADD += ${top_builddir}/topography/libtopography.la + libFMS_la_LIBADD += ${top_builddir}/tracer_manager/libtracer_manager.la + libFMS_la_LIBADD += ${top_builddir}/station_data/libstation_data.la ++libFMS_la_LIBADD += ${top_builddir}/sat_vapor_pres/libsat_vapor_pres.la ++libFMS_la_LIBADD += ${top_builddir}/sat_vapor_pres/libsat_vapor_pres_k.la + + # At least one source file must be included to please Automake. + libFMS_la_SOURCES = ${top_builddir}/include/file_version.h +-- +2.24.0 + diff --git a/test/patches/PATCH-fv3atm-apply-fixes.patch b/test/patches/PATCH-fv3atm-apply-fixes.patch new file mode 100644 index 000000000..36faec273 --- /dev/null +++ b/test/patches/PATCH-fv3atm-apply-fixes.patch @@ -0,0 +1,560 @@ +From 00f54c3119f77d3cd09180c608eb51719c85455f Mon Sep 17 00:00:00 2001 +From: Oliver Fuhrer +Date: Fri, 8 May 2020 14:49:55 -0700 +Subject: [PATCH] fix fv3atm + +--- + coupler_main.F90 | 510 +++++++++++++++++++++++++++++++++++++++++++++++ + makefile | 6 +- + 2 files changed, 513 insertions(+), 3 deletions(-) + create mode 100644 coupler_main.F90 + +diff --git a/coupler_main.F90 b/coupler_main.F90 +new file mode 100644 +index 0000000..2efb8e5 +--- /dev/null ++++ b/coupler_main.F90 +@@ -0,0 +1,510 @@ ++!*********************************************************************** ++!* GNU General Public License * ++!* This file is a part of fvGFS. * ++!* * ++!* fvGFS is free software; you can redistribute it and/or modify it * ++!* and are expected to follow the terms of the GNU General Public * ++!* License as published by the Free Software Foundation; either * ++!* version 2 of the License, or (at your option) any later version. * ++!* * ++!* fvGFS is distributed in the hope that it will be useful, but * ++!* WITHOUT ANY WARRANTY; without even the implied warranty of * ++!* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * ++!* General Public License for more details. * ++!* * ++!* For the full text of the GNU General Public License, * ++!* write to: Free Software Foundation, Inc., * ++!* 675 Mass Ave, Cambridge, MA 02139, USA. * ++!* or see: http://www.gnu.org/licenses/gpl.html * ++!*********************************************************************** ++ ++program coupler_main ++ ++!----------------------------------------------------------------------- ++! ++! program that couples component models for the atmosphere, ++! ocean (amip), land, and sea-ice using the exchange module ++! ++!----------------------------------------------------------------------- ++!$ser verbatim use mpi ++use time_manager_mod, only: time_type, set_calendar_type, set_time, & ++ set_date, days_in_month, month_name, & ++ operator(+), operator (<), operator (>), & ++ operator (/=), operator (/), operator (==),& ++ operator (*), THIRTY_DAY_MONTHS, JULIAN, & ++ NOLEAP, NO_CALENDAR, date_to_string, & ++ get_date ++ ++use atmos_model_mod, only: atmos_model_init, atmos_model_end, & ++ update_atmos_model_dynamics, & ++ update_atmos_radiation_physics, & ++ update_atmos_model_state, & ++ atmos_data_type, atmos_model_restart ++ ++use constants_mod, only: constants_init ++#ifdef INTERNAL_FILE_NML ++use mpp_mod, only: input_nml_file ++#else ++use fms_mod, only: open_namelist_file ++#endif ++use fms_mod, only: file_exist, check_nml_error, & ++ error_mesg, fms_init, fms_end, close_file, & ++ write_version_number, uppercase ++ ++use mpp_mod, only: mpp_init, mpp_pe, mpp_root_pe, mpp_npes, mpp_get_current_pelist, & ++ mpp_set_current_pelist, stdlog, mpp_error, NOTE, FATAL, WARNING ++use mpp_mod, only: mpp_clock_id, mpp_clock_begin, mpp_clock_end, mpp_sync ++ ++use mpp_io_mod, only: mpp_open, mpp_close, & ++ MPP_NATIVE, MPP_RDONLY, MPP_DELETE ++ ++use mpp_domains_mod, only: mpp_get_global_domain, mpp_global_field, CORNER ++use memutils_mod, only: print_memuse_stats ++use sat_vapor_pres_mod,only: sat_vapor_pres_init ++ ++use diag_manager_mod, only: diag_manager_init, diag_manager_end, & ++ get_base_date, diag_manager_set_time_end ++ ++use data_override_mod, only: data_override_init ++ ++!$ser verbatim use,intrinsic :: ISO_Fortran_env ++implicit none ++ ++!----------------------------------------------------------------------- ++ ++character(len=128) :: version = '$Id: coupler_main.F90,v 19.0.4.1.2.3 2014/09/09 23:51:59 Rusty.Benson Exp $' ++character(len=128) :: tag = '$Name: ulm_201505 $' ++ ++!----------------------------------------------------------------------- ++!---- model defined-types ---- ++ ++ type (atmos_data_type) :: Atm ++ ++!----------------------------------------------------------------------- ++! ----- coupled model time ----- ++ ++ type (time_type) :: Time_atmos, Time_init, Time_end, & ++ Time_step_atmos, Time_step_ocean, & ++ Time_restart, Time_step_restart ++ integer :: num_cpld_calls, num_atmos_calls, nc, na, ret ++ ++! ----- coupled model initial date ----- ++ ++ integer :: date_init(6) ++ integer :: calendar_type = -99 ++ ++! ----- timing flags ----- ++ ++ integer :: initClock, mainClock, termClock ++ integer, parameter :: timing_level = 1 ++ ++! ----- namelist ----- ++ integer, dimension(6) :: current_date = (/ 0, 0, 0, 0, 0, 0 /) ++ character(len=17) :: calendar = ' ' ++ logical :: force_date_from_namelist = .false. ! override restart values for date ++ integer :: months=0, days=0, hours=0, minutes=0, seconds=0 ++ integer :: dt_atmos = 0 ++ integer :: dt_ocean = 0 ++ integer :: restart_days = 0 ++ integer :: restart_secs = 0 ++ integer :: atmos_nthreads = 1 ++ logical :: memuse_verbose = .false. ++ logical :: use_hyper_thread = .false. ++ logical :: debug_affinity = .false. ++ integer :: ncores_per_node = 0 ++ ++ namelist /coupler_nml/ current_date, calendar, force_date_from_namelist, & ++ months, days, hours, minutes, seconds, & ++ dt_atmos, dt_ocean, atmos_nthreads, memuse_verbose, & ++ use_hyper_thread, ncores_per_node, debug_affinity, & ++ restart_secs, restart_days ++ ++! ----- local variables ----- ++ character(len=32) :: timestamp ++ logical :: intrm_rst ++ !$ser verbatim integer :: mpi_rank,ier ++ ++!####################################################################### ++ ++ call fms_init() ++ call mpp_init() ++ initClock = mpp_clock_id( 'Initialization' ) ++ call mpp_clock_begin (initClock) !nesting problem ++ ++ call fms_init ++ call constants_init ++ call sat_vapor_pres_init ++ !$ser verbatim call mpi_comm_rank(MPI_COMM_WORLD, mpi_rank,ier) ++ !$ser init directory='.' prefix='Generator' mpi_rank=mpi_rank unique_id=.true. ++ !$ser mode write ++ !$ser off ++ call coupler_init ++ call print_memuse_stats('after coupler init') ++ ++ call mpp_set_current_pelist() ++ call mpp_clock_end (initClock) !end initialization ++ mainClock = mpp_clock_id( 'Main loop' ) ++ termClock = mpp_clock_id( 'Termination' ) ++ call mpp_clock_begin(mainClock) !begin main loop ++ ++ do nc = 1, num_cpld_calls ++ ++ Time_atmos = Time_atmos + Time_step_atmos ++ ++ call update_atmos_model_dynamics (Atm) ++ ++ call update_atmos_radiation_physics (Atm) ++ ++ call update_atmos_model_state (Atm) ++ ++!--- intermediate restart ++ if (intrm_rst) then ++ if ((nc /= num_cpld_calls) .and. (Time_atmos == Time_restart)) then ++ timestamp = date_to_string (Time_restart) ++ call atmos_model_restart(Atm, timestamp) ++ call coupler_res(timestamp) ++ Time_restart = Time_restart + Time_step_restart ++ endif ++ endif ++ ++ call print_memuse_stats('after full step') ++ ++ enddo ++ !$ser cleanup ++!----------------------------------------------------------------------- ++ ++#ifdef AVEC_TIMERS ++ call avec_timers_output ++#endif ++ call mpp_set_current_pelist() ++ call mpp_clock_end(mainClock) ++ call mpp_clock_begin(termClock) ++ ++ call coupler_end ++ call mpp_set_current_pelist() ++ call mpp_clock_end(termClock) ++ ++ call fms_end ++ ++!----------------------------------------------------------------------- ++ ++ stop ++ ++contains ++ ++!####################################################################### ++ ++ subroutine coupler_init ++ ++!----------------------------------------------------------------------- ++! initialize all defined exchange grids and all boundary maps ++!----------------------------------------------------------------------- ++ integer :: total_days, total_seconds, unit, ierr, io ++ integer :: n, gnlon, gnlat ++ integer :: date(6), flags ++ type (time_type) :: Run_length ++ character(len=9) :: month ++ logical :: use_namelist ++ ++ logical, allocatable, dimension(:,:) :: mask ++ real, allocatable, dimension(:,:) :: glon_bnd, glat_bnd ++ integer :: omp_get_thread_num, get_cpu_affinity, base_cpu ++!----------------------------------------------------------------------- ++!----- initialization timing identifiers ---- ++ ++!----- read namelist ------- ++!----- for backwards compatibilty read from file coupler.nml ----- ++ ++#ifdef INTERNAL_FILE_NML ++ read(input_nml_file, nml=coupler_nml, iostat=io) ++ ierr = check_nml_error(io, 'coupler_nml') ++#else ++ if (file_exist('input.nml')) then ++ unit = open_namelist_file () ++ else ++ call error_mesg ('program coupler', & ++ 'namelist file input.nml does not exist', FATAL) ++ endif ++ ++ ierr=1 ++ do while (ierr /= 0) ++ read (unit, nml=coupler_nml, iostat=io, end=10) ++ ierr = check_nml_error (io, 'coupler_nml') ++ enddo ++10 call close_file (unit) ++#endif ++ ++!----- write namelist to logfile ----- ++ call write_version_number (version, tag) ++ if (mpp_pe() == mpp_root_pe()) write(stdlog(),nml=coupler_nml) ++ ++!----- allocate and set the pelist (to the global pelist) ----- ++ allocate( Atm%pelist (mpp_npes()) ) ++ call mpp_get_current_pelist(Atm%pelist) ++ ++!----- read restart file ----- ++ ++ if (file_exist('INPUT/coupler.res')) then ++ call mpp_open( unit, 'INPUT/coupler.res', action=MPP_RDONLY ) ++ read (unit,*,err=999) calendar_type ++ read (unit,*) date_init ++ read (unit,*) date ++ goto 998 !back to fortran-4 ++ ! read old-style coupler.res ++ 999 call mpp_close (unit) ++ call mpp_open (unit, 'INPUT/coupler.res', action=MPP_RDONLY, form=MPP_NATIVE) ++ read (unit) calendar_type ++ read (unit) date ++ 998 call mpp_close(unit) ++ else ++ force_date_from_namelist = .true. ++ endif ++ ++!----- use namelist value (either no restart or override flag on) --- ++ ++ if ( force_date_from_namelist ) then ++ ++ if ( sum(current_date) <= 0 ) then ++ call error_mesg ('program coupler', & ++ 'no namelist value for current_date', FATAL) ++ else ++ date = current_date ++ endif ++ ++!----- override calendar type with namelist value ----- ++ ++ select case( uppercase(trim(calendar)) ) ++ case( 'JULIAN' ) ++ calendar_type = JULIAN ++ case( 'NOLEAP' ) ++ calendar_type = NOLEAP ++ case( 'THIRTY_DAY' ) ++ calendar_type = THIRTY_DAY_MONTHS ++ case( 'NO_CALENDAR' ) ++ calendar_type = NO_CALENDAR ++ case default ++ call mpp_error ( FATAL, 'COUPLER_MAIN: coupler_nml entry calendar must '// & ++ 'be one of JULIAN|NOLEAP|THIRTY_DAY|NO_CALENDAR.' ) ++ end select ++ ++ endif ++ ++!$ base_cpu = get_cpu_affinity() ++!$ call omp_set_num_threads(atmos_nthreads) ++!$OMP PARALLEL NUM_THREADS(atmos_nthreads) ++!$ if(omp_get_thread_num() < atmos_nthreads/2 .OR. (.not. use_hyper_thread)) then ++!$ call set_cpu_affinity(base_cpu + omp_get_thread_num()) ++!$ else ++!$ call set_cpu_affinity(base_cpu + omp_get_thread_num() + & ++!$ ncores_per_node - atmos_nthreads/2) ++!$ endif ++!$ if (debug_affinity) then ++!$ write(6,*) mpp_pe()," atmos ",get_cpu_affinity(), base_cpu, omp_get_thread_num() ++!$ call flush(6) ++!$ endif ++!$OMP END PARALLEL ++ ++ call set_calendar_type (calendar_type) ++ ++!----- write current/initial date actually used to logfile file ----- ++ ++ if ( mpp_pe() == mpp_root_pe() ) then ++ write (stdlog(),16) date(1),trim(month_name(date(2))),date(3:6) ++ endif ++ ++ 16 format (' current date used = ',i4,1x,a,2i3,2(':',i2.2),' gmt') ++ ++!----------------------------------------------------------------------- ++!------ initialize diagnostics manager ------ ++ ++ call diag_manager_init (TIME_INIT=date) ++ ++!----- always override initial/base date with diag_manager value ----- ++ ++ call get_base_date ( date_init(1), date_init(2), date_init(3), & ++ date_init(4), date_init(5), date_init(6) ) ++ ++!----- use current date if no base date ------ ++ ++ if ( date_init(1) == 0 ) date_init = date ++ ++!----- set initial and current time types ------ ++ ++ Time_init = set_date (date_init(1), date_init(2), date_init(3), & ++ date_init(4), date_init(5), date_init(6)) ++ ++ Time_atmos = set_date (date(1), date(2), date(3), & ++ date(4), date(5), date(6)) ++ ++!----------------------------------------------------------------------- ++!----- compute the ending time (compute days in each month first) ----- ++! ++! (NOTE: if run length in months then starting day must be <= 28) ++ ++ if ( months > 0 .and. date(3) > 28 ) & ++ call error_mesg ('program coupler', & ++ 'if run length in months then starting day must be <= 28', FATAL) ++ ++ Time_end = Time_atmos ++ total_days = 0 ++ do n = 1, months ++ total_days = total_days + days_in_month(Time_end) ++ Time_end = Time_atmos + set_time (0,total_days) ++ enddo ++ ++ total_days = total_days + days ++ total_seconds = hours*3600 + minutes*60 + seconds ++ Run_length = set_time (total_seconds,total_days) ++ Time_end = Time_atmos + Run_length ++ ++ !Need to pass Time_end into diag_manager for multiple thread case. ++ call diag_manager_set_time_end(Time_end) ++ ++ ++!----------------------------------------------------------------------- ++!----- write time stamps (for start time and end time) ------ ++ ++ call mpp_open( unit, 'time_stamp.out', nohdrs=.TRUE. ) ++ ++ month = month_name(date(2)) ++ if ( mpp_pe() == mpp_root_pe() ) write (unit,20) date, month(1:3) ++ ++ call get_date (Time_end, date(1), date(2), date(3), & ++ date(4), date(5), date(6)) ++ month = month_name(date(2)) ++ if ( mpp_pe() == mpp_root_pe() ) write (unit,20) date, month(1:3) ++ ++ call mpp_close (unit) ++ ++ 20 format (6i4,2x,a3) ++ ++!----------------------------------------------------------------------- ++!----- compute the time steps ------ ++ ++Time_step_atmos = set_time (dt_atmos,0) ++Time_step_ocean = set_time (dt_ocean,0) ++num_cpld_calls = Run_length / Time_step_ocean ++num_atmos_calls = Time_step_ocean / Time_step_atmos ++Time_step_restart = set_time (restart_secs, restart_days) ++Time_restart = Time_atmos + Time_step_restart ++intrm_rst = .false. ++if (restart_days > 0 .or. restart_secs > 0) intrm_rst = .true. ++ ++!----------------------------------------------------------------------- ++!------------------- some error checks --------------------------------- ++ ++!----- initial time cannot be greater than current time ------- ++ ++ if ( Time_init > Time_atmos ) call error_mesg ('program coupler', & ++ 'initial time is greater than current time', FATAL) ++ ++!----- make sure run length is a multiple of ocean time step ------ ++ ++ if ( num_cpld_calls * Time_step_ocean /= Run_length ) & ++ call error_mesg ('program coupler', & ++ 'run length must be multiple of ocean time step', FATAL) ++ ++! ---- make sure cpld time step is a multiple of atmos time step ---- ++ ++ if ( num_atmos_calls * Time_step_atmos /= Time_step_ocean ) & ++ call error_mesg ('program coupler', & ++ 'atmos time step is not a multiple of the ocean time step', FATAL) ++ ++!------ initialize component models ------ ++ ++ call atmos_model_init (Atm, Time_init, Time_atmos, Time_step_atmos) ++ ++ call print_memuse_stats('after atmos model init') ++ ++ call mpp_get_global_domain(Atm%Domain, xsize=gnlon, ysize=gnlat) ++ allocate ( glon_bnd(gnlon+1,gnlat+1), glat_bnd(gnlon+1,gnlat+1) ) ++ call mpp_global_field(Atm%Domain, Atm%lon_bnd, glon_bnd, position=CORNER) ++ call mpp_global_field(Atm%Domain, Atm%lat_bnd, glat_bnd, position=CORNER) ++ ++ call data_override_init ( ) ! Atm_domain_in = Atm%domain, & ++ ! Ice_domain_in = Ice%domain, & ++ ! Land_domain_in = Land%domain ) ++ ++!----------------------------------------------------------------------- ++!---- open and close dummy file in restart dir to check if dir exists -- ++ ++ if (mpp_pe() == 0 ) then ++ call mpp_open( unit, 'RESTART/file' ) ++ call mpp_close(unit, MPP_DELETE) ++ endif ++ ++!----------------------------------------------------------------------- ++ ++ end subroutine coupler_init ++ ++!####################################################################### ++ subroutine coupler_res(timestamp) ++ character(len=32), intent(in) :: timestamp ++ ++ integer :: unit, date(6) ++ ++!----- compute current date ------ ++ ++ call get_date (Time_atmos, date(1), date(2), date(3), & ++ date(4), date(5), date(6)) ++ ++!----- write restart file ------ ++ ++ if (mpp_pe() == mpp_root_pe())then ++ call mpp_open( unit, 'RESTART/'//trim(timestamp)//'.coupler.res', nohdrs=.TRUE. ) ++ write( unit, '(i6,8x,a)' )calendar_type, & ++ '(Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)' ++ ++ write( unit, '(6i6,8x,a)' )date_init, & ++ 'Model start time: year, month, day, hour, minute, second' ++ write( unit, '(6i6,8x,a)' )date, & ++ 'Current model time: year, month, day, hour, minute, second' ++ call mpp_close(unit) ++ endif ++ end subroutine coupler_res ++ ++!####################################################################### ++ ++ subroutine coupler_end ++ ++ integer :: unit, date(6) ++!----------------------------------------------------------------------- ++ ++ call atmos_model_end (Atm) ++ ++!----- compute current date ------ ++ ++ call get_date (Time_atmos, date(1), date(2), date(3), & ++ date(4), date(5), date(6)) ++ ++!----- check time versus expected ending time ---- ++ ++ if (Time_atmos /= Time_end) call error_mesg ('program coupler', & ++ 'final time does not match expected ending time', WARNING) ++ ++!----- write restart file ------ ++ ++ call mpp_open( unit, 'RESTART/coupler.res', nohdrs=.TRUE. ) ++ if (mpp_pe() == mpp_root_pe())then ++ write( unit, '(i6,8x,a)' )calendar_type, & ++ '(Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)' ++ ++ write( unit, '(6i6,8x,a)' )date_init, & ++ 'Model start time: year, month, day, hour, minute, second' ++ write( unit, '(6i6,8x,a)' )date, & ++ 'Current model time: year, month, day, hour, minute, second' ++ endif ++ call mpp_close(unit) ++ ++!----- final output of diagnostic fields ---- ++ ++ call diag_manager_end (Time_atmos) ++ ++!----------------------------------------------------------------------- ++ ++ end subroutine coupler_end ++ ++!####################################################################### ++ ++end program coupler_main ++ +diff --git a/makefile b/makefile +index 104e885..1fddc54 100644 +--- a/makefile ++++ b/makefile +@@ -41,7 +41,7 @@ libs: + $(MAKE) -C atmos_cubed_sphere $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) + $(MAKE) -C ../stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit + +-$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a ++$(FV3_EXE): atmos_model.o coupler_main.o module_fv3_config.o module_fv3_config.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a + $(LD) -o $@ $^ $(NCEPLIBS) $(LDFLAGS) + + else +@@ -53,14 +53,14 @@ libs: + $(MAKE) -C atmos_cubed_sphere $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) + $(MAKE) -C ../stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit + +-$(FV3_EXE): atmos_model.o coupler_main.o atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a ++$(FV3_EXE): atmos_model.o coupler_main.o module_fv3_config.o atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a + $(LD) -o $@ $^ $(NCEPLIBS) $(LDFLAGS) + endif + + $(FV3CAP_LIB): atmos_model.o module_fv3_config.o module_fcst_grid_comp.o time_utils.o fv3_cap.o + ar rv $(FV3CAP_LIB) $? + +-atmos_model.o : atmos_model.F90 ++atmos_model.o : atmos_model.F90 module_fv3_config.o + $(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c atmos_model.F90 + + module_fv3_config.o: module_fv3_config.F90 +-- +2.24.0 + diff --git a/test/patches/PATCH-stochastic_physics-update_ca.F90-fix-logical-comparisons.patch b/test/patches/PATCH-stochastic_physics-update_ca.F90-fix-logical-comparisons.patch new file mode 100644 index 000000000..4399db598 --- /dev/null +++ b/test/patches/PATCH-stochastic_physics-update_ca.F90-fix-logical-comparisons.patch @@ -0,0 +1,25 @@ +From fe715358a087d1073777bab79740dd1d50a99231 Mon Sep 17 00:00:00 2001 +From: Oliver Fuhrer +Date: Fri, 8 May 2020 11:13:42 -0700 +Subject: [PATCH] fix logical comparison + +--- + update_ca.F90 | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/update_ca.F90 b/update_ca.F90 +index 55d5305..05b0b6f 100644 +--- a/update_ca.F90 ++++ b/update_ca.F90 +@@ -243,7 +243,7 @@ enddo !spinup + incj=incj+ncells + ENDDO + +-if(nca_plumes == .true.) then ++if(nca_plumes) then + !COMPUTE NUMBER OF CLUSTERS (CONVECTIVE PLUMES) IN EACH CA-CELL + !Note, at the moment we only use the count of the plumes found in a grid-cell + !In the future the routine "plumes" can also be used to give the size of +-- +2.24.0 + diff --git a/test/work/run_test.sh b/test/work/run_test.sh new file mode 100755 index 000000000..d802269e6 --- /dev/null +++ b/test/work/run_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -x +set -e + +cd /work + +if [ -d ./rundir ] ; then + echo "ERROR: rundir already exists" + exit 1 +fi + +#./create_rundir.py +curl -X GET 'https://www.googleapis.com/storage/v1/b/vcm-ml-public/o/data_for_demos%2FTravisCI%2Frundir.tar.gz?alt=media' | tar xvz + +if [ ! -d ./rundir -o ! -f ./rundir/INPUT/gfs_data.tile1.nc ] ; then + echo "ERROR: problem creating rundir" + exit 1 +fi + +cd /work/rundir +ln -s /FV3/fv3.exe . +mpirun -n 6 --allow-run-as-root --mca btl_vader_single_copy_mechanism none \ + --oversubscribe --merge-stderr-to-stdout --tag-output --timestamp-output \ + ./fv3.exe 2>&1 | tee fv3.out + +exit 0