From d6e294f94bae614f0372a87c14d00a26b2a8c07e Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Tue, 16 Jan 2024 16:21:40 -0500 Subject: [PATCH 1/3] Create BUILD and WORK directory macros This patch replaces the WORKSPACE macro with two macros for BUILD and WORK. It also makes the following smaller changes: * deps/ is now defined by the DEPS macro. If unset, deps/ is placed in the BUILD directory. * results/ is moved into WORK, * Compiler flags which track directories now use $(abspath ...) to allow for arbitrary paths. * GitHub CI paths were adjusted to support these new settings. This work will enable testing of multiple FMS libraries in our CI. --- .github/actions/testing-setup/action.yml | 2 +- .testing/Makefile | 303 ++++++++++++----------- 2 files changed, 153 insertions(+), 152 deletions(-) diff --git a/.github/actions/testing-setup/action.yml b/.github/actions/testing-setup/action.yml index a15dd6d0a2..60499d4be1 100644 --- a/.github/actions/testing-setup/action.yml +++ b/.github/actions/testing-setup/action.yml @@ -28,7 +28,7 @@ runs: run: | echo "::group::Compile FMS library" cd .testing - REPORT_ERROR_LOGS=true make deps/lib/libFMS.a -s -j + REPORT_ERROR_LOGS=true make build/deps/lib/libFMS.a -s -j echo "::endgroup::" - name: Compile MOM6 in symmetric memory mode diff --git a/.testing/Makefile b/.testing/Makefile index f7101a3463..400081fd9e 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -46,10 +46,10 @@ # LDFLAGS_USER User-defined linker flags (used for all MOM/FMS builds) # # Experiment Configuration: -# BUILDS Executables to be built by `make` or `make all` +# EXECS Executables to be built by `make` or `make all` # CONFIGS Model configurations to test (default: `tc*`) -# TESTS Tests to run # DIMS Dimensional scaling tests +# TESTS Tests to run # # Regression repository ("target") configuration: # MOM_TARGET_SLUG URL slug (minus domain) of the target repo @@ -57,8 +57,10 @@ # MOM_TARGET_LOCAL_BRANCH Target branch name # (NOTE: These would typically be configured by a CI.) # -# Paths for stages: -# WORKSPACE Location to place work/ and results/ directories (i.e. where to run the model) +# Output paths: +# BUILD Compiled executables and libraries +# DEPS Compiled dependencies +# WORK Test model output # #---- @@ -128,15 +130,17 @@ REPORT_ERROR_LOGS ?= # Time measurement (configurable by the CI) TIME ?= time - # Experiment configuration -BUILDS ?= symmetric/MOM6 asymmetric/MOM6 openmp/MOM6 +EXECS ?= symmetric/MOM6 asymmetric/MOM6 openmp/MOM6 CONFIGS ?= $(wildcard tc*) -TESTS ?= grid layout rotate restart openmp nan $(foreach d,$(DIMS),dim.$(d)) DIMS ?= t l h z q r +TESTS ?= grid layout rotate restart openmp nan $(foreach d,$(DIMS),dim.$(d)) # Default is to place work/ and results/ in current directory -WORKSPACE ?= . +BUILD ?= build +DEPS ?= $(BUILD)/deps +WORK ?= work + #--- # Test configuration @@ -145,18 +149,18 @@ WORKSPACE ?= . # the DEBUG results in older GCCs and vendor compilers, so we can optionally # disable them. ifeq ($(DO_REPRO_TESTS), true) - BUILDS += repro/MOM6 + EXECS += repro/MOM6 TESTS += repro endif # Profiling ifeq ($(DO_PROFILE), true) - BUILDS += opt/MOM6 opt_target/MOM6 + EXECS += opt/MOM6 opt_target/MOM6 endif # Coverage ifeq ($(DO_COVERAGE), true) - BUILDS += cov/MOM6 + EXECS += cov/MOM6 endif # Unit testing (or coverage) @@ -167,13 +171,12 @@ ifneq (X$(DO_COVERAGE)$(DO_UNIT_TESTS)X, XX) endif ifeq ($(DO_PROFILE), false) - BUILDS += opt/MOM6 opt_target/MOM6 + EXECS += opt/MOM6 opt_target/MOM6 endif - DO_REGRESSION_TESTS ?= ifeq ($(DO_REGRESSION_TESTS), true) - BUILDS += target/MOM6 + EXECS += target/MOM6 TESTS += regression MOM_TARGET_SLUG ?= NOAA-GFDL/MOM6 @@ -182,13 +185,14 @@ ifeq ($(DO_REGRESSION_TESTS), true) MOM_TARGET_LOCAL_BRANCH ?= dev/gfdl MOM_TARGET_BRANCH := origin/$(MOM_TARGET_LOCAL_BRANCH) - TARGET_CODEBASE = build/target_codebase + TARGET_CODEBASE = $(BUILD)/target_codebase else MOM_TARGET_URL = MOM_TARGET_BRANCH = TARGET_CODEBASE = endif + # List of source files to link this Makefile's dependencies to model Makefiles # Assumes a depth of two, and the following extensions: F90 inc c h # (1): Root directory @@ -202,31 +206,30 @@ MOM_SOURCE = \ $(wildcard ../config_src/ext*/*/*.F90) TARGET_SOURCE = \ - $(call SOURCE,build/target_codebase/src) \ - $(wildcard build/target_codebase/config_src/drivers/solo_driver/*.F90) \ - $(wildcard build/target_codebase/config_src/ext*/*.F90) + $(call SOURCE,$(BUILD)/target_codebase/src) \ + $(wildcard $(BUILD)/target_codebase/config_src/drivers/solo_driver/*.F90) \ + $(wildcard $(BUILD)target_codebase/config_src/ext*/*.F90) -# NOTE: Current default framework is FMS1, but this could change. ifeq ($(FRAMEWORK), fms2) MOM_SOURCE +=$(wildcard ../config_src/infra/FMS2/*.F90) - TARGET_SOURCE += $(wildcard build/target_codebase/config_src/infra/FMS2/*.F90) + TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS2/*.F90) else MOM_SOURCE += $(wildcard ../config_src/infra/FMS1/*.F90) - TARGET_SOURCE += $(wildcard build/target_codebase/config_src/infra/FMS1/*.F90) + TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS1/*.F90) endif -FMS_SOURCE = $(call SOURCE,deps/fms/src) +FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src) #--- # Rules .PHONY: all build.regressions build.prof -all: $(foreach b,$(BUILDS),build/$(b)) -build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6) -build.prof: $(foreach b,opt opt_target,build/$(b)/MOM6) +all: $(foreach b,$(EXECS),$(BUILD)/$(b)) +build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6) +build.prof: $(foreach b,opt opt_target,$(BUILD)/$(b)/MOM6) # Executable -.PRECIOUS: $(foreach b,$(BUILDS),build/$(b)) +.PRECIOUS: $(foreach b,$(EXECS),$(BUILD)/$(b)) # Compiler flags @@ -234,9 +237,9 @@ build.prof: $(foreach b,opt opt_target,build/$(b)/MOM6) # .testing dependencies # TODO: We should probably build TARGET with the FMS that it was configured # to use. But for now we use the same FMS over all builds. -FCFLAGS_DEPS = -I../../deps/include -LDFLAGS_DEPS = -L../../deps/lib -PATH_DEPS = PATH="${PATH}:../../deps/bin" +FCFLAGS_DEPS = -I$(abspath $(DEPS)/include) +LDFLAGS_DEPS = -L$(abspath $(DEPS)/lib) +PATH_DEPS = PATH="${PATH}:$(abspath $(DEPS)/bin)" # Define the build targets in terms of the traditional DEBUG/REPRO/etc labels @@ -254,75 +257,72 @@ COV_LDFLAGS := LDFLAGS="$(LDFLAGS_COVERAGE) $(LDFLAGS_DEPS) $(LDFLAGS_USER)" # Environment variable configuration MOM_ENV := $(PATH_FMS) -build/symmetric/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) -build/asymmetric/Makefile: MOM_ENV += $(ASYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) \ +$(BUILD)/symmetric/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/asymmetric/Makefile: MOM_ENV += $(ASYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) \ MOM_MEMORY=../../../config_src/memory/dynamic_nonsymmetric/MOM_memory.h -build/repro/Makefile: MOM_ENV += $(REPRO_FCFLAGS) $(MOM_LDFLAGS) -build/openmp/Makefile: MOM_ENV += $(OPENMP_FCFLAGS) $(MOM_LDFLAGS) -build/target/Makefile: MOM_ENV += $(TARGET_FCFLAGS) $(MOM_LDFLAGS) -build/opt/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) -build/opt_target/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) -build/coupled/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) -build/nuopc/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) -build/cov/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS) -build/unit/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS) -build/timing/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/repro/Makefile: MOM_ENV += $(REPRO_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/openmp/Makefile: MOM_ENV += $(OPENMP_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/target/Makefile: MOM_ENV += $(TARGET_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/opt/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/opt_target/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/coupled/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/nuopc/Makefile: MOM_ENV += $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS) +$(BUILD)/cov/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS) +$(BUILD)/unit/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS) +$(BUILD)/timing/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS) # Configure script flags MOM_ACFLAGS := --with-framework=$(FRAMEWORK) -build/openmp/Makefile: MOM_ACFLAGS += --enable-openmp -build/coupled/Makefile: MOM_ACFLAGS += --with-driver=FMS_cap -build/nuopc/Makefile: MOM_ACFLAGS += --with-driver=nuopc_cap -build/unit/Makefile: MOM_ACFLAGS += --with-driver=unit_tests -build/timing/Makefile: MOM_ACFLAGS += --with-driver=timing_tests +$(BUILD)/openmp/Makefile: MOM_ACFLAGS += --enable-openmp +$(BUILD)/coupled/Makefile: MOM_ACFLAGS += --with-driver=FMS_cap +$(BUILD)/nuopc/Makefile: MOM_ACFLAGS += --with-driver=nuopc_cap +$(BUILD)/unit/Makefile: MOM_ACFLAGS += --with-driver=unit_tests +$(BUILD)/timing/Makefile: MOM_ACFLAGS += --with-driver=timing_tests # Fetch regression target source code -build/target/Makefile: | $(TARGET_CODEBASE) -build/opt_target/Makefile: | $(TARGET_CODEBASE) +$(BUILD)/target/Makefile: | $(TARGET_CODEBASE) +$(BUILD)/opt_target/Makefile: | $(TARGET_CODEBASE) # Define source code dependencies -build/target_codebase/configure: $(TARGET_SOURCE) +$(BUILD)/target_codebase/configure: $(TARGET_SOURCE) # Build executables -build/unit/test_%: build/unit/Makefile FORCE +$(BUILD)/unit/test_%: $(BUILD)/unit/Makefile FORCE cd $(@D) && $(TIME) $(MAKE) $(@F) -j -build/unit/Makefile: $(foreach e,$(UNIT_EXECS),../config_src/drivers/unit_tests/$(e).F90) -build/timing/time_%: build/timing/Makefile FORCE +$(BUILD)/unit/Makefile: $(foreach e,$(UNIT_EXECS),../config_src/drivers/unit_tests/$(e).F90) +$(BUILD)/timing/time_%: $(BUILD)/timing/Makefile FORCE cd $(@D) && $(TIME) $(MAKE) $(@F) -j -build/timing/Makefile: $(foreach e,$(TIMING_EXECS),../config_src/drivers/timing_tests/$(e).F90) -build/%/MOM6: build/%/Makefile FORCE +$(BUILD)/timing/Makefile: $(foreach e,$(TIMING_EXECS),../config_src/drivers/timing_tests/$(e).F90) +$(BUILD)/%/MOM6: $(BUILD)/%/Makefile FORCE cd $(@D) && $(TIME) $(MAKE) $(@F) -j -FORCE: ; + +FORCE: # Use autoconf to construct the Makefile for each target -.PRECIOUS: build/%/Makefile -build/%/Makefile: ../ac/configure ../ac/Makefile.in deps/lib/libFMS.a +.PRECIOUS: $(BUILD)/%/Makefile +$(BUILD)/%/Makefile: ../ac/configure ../ac/Makefile.in $(DEPS)/lib/libFMS.a mkdir -p $(@D) cd $(@D) \ && $(MOM_ENV) ../../../ac/configure $(MOM_ACFLAGS) \ || (cat config.log && false) - ../ac/configure: ../ac/configure.ac ../ac/m4 autoreconf -i $< - # Fetch the regression target codebase -build/target/Makefile build/opt_target/Makefile: \ - $(TARGET_CODEBASE)/ac/configure deps/lib/libFMS.a +$(BUILD)/target/Makefile $(BUILD)/opt_target/Makefile: \ + $(TARGET_CODEBASE)/ac/configure $(DEPS)/lib/libFMS.a mkdir -p $(@D) cd $(@D) \ && $(MOM_ENV) ../../$(TARGET_CODEBASE)/ac/configure $(MOM_ACFLAGS) \ || (cat config.log && false) - $(TARGET_CODEBASE)/ac/configure: $(TARGET_CODEBASE) autoreconf -i $/dev/null)" || rm -rf $(WORKSPACE)/results/$(1) +.PRECIOUS: $(foreach b,$(3),$(WORK)/$(1)/$(b)/ocean.stats) +$(1).$(2): $(foreach b,$(3),$(WORK)/$(1)/$(b)/ocean.stats) + @test "$$(shell ls -A $(WORK)/results/$(1) 2>/dev/null)" || rm -rf $(WORK)/results/$(1) @cmp $$^ || !( \ - mkdir -p $(WORKSPACE)/results/$(1); \ - (diff $$^ | tee $(WORKSPACE)/results/$(1)/ocean.stats.$(2).diff | head -n 20) ; \ + mkdir -p $(WORK)/results/$(1); \ + (diff $$^ | tee $(WORK)/results/$(1)/ocean.stats.$(2).diff | head -n 20) ; \ echo -e "$(FAIL): Solutions $(1).$(2) have changed." \ ) @echo -e "$(PASS): Solutions $(1).$(2) agree." -.PRECIOUS: $(foreach b,$(3),$(WORKSPACE)/work/$(1)/$(b)/chksum_diag) -$(1).$(2).diag: $(foreach b,$(3),$(WORKSPACE)/work/$(1)/$(b)/chksum_diag) +.PRECIOUS: $(foreach b,$(3),$(WORK)/$(1)/$(b)/chksum_diag) +$(1).$(2).diag: $(foreach b,$(3),$(WORK)/$(1)/$(b)/chksum_diag) @cmp $$^ || !( \ - mkdir -p $(WORKSPACE)/results/$(1); \ - (diff $$^ | tee $(WORKSPACE)/results/$(1)/chksum_diag.$(2).diff | head -n 20) ; \ + mkdir -p $(WORK)/results/$(1); \ + (diff $$^ | tee $(WORK)/results/$(1)/chksum_diag.$(2).diff | head -n 20) ; \ echo -e "$(FAIL): Diagnostics $(1).$(2).diag have changed." \ ) @echo -e "$(PASS): Diagnostics $(1).$(2).diag agree." @@ -473,17 +474,17 @@ $(foreach d,$(DIMS),$(eval $(call CMP_RULE,$(1),dim.$(d),symmetric dim.$(d)))) endef $(foreach c,$(CONFIGS),$(eval $(call CONFIG_DIM_RULE,$(c)))) -# Custom comparison rules +# Custom comparison rules # Restart tests only compare the final stat record -.PRECIOUS: $(foreach b,symmetric restart target,$(WORKSPACE)/work/%/$(b)/ocean.stats) -%.restart: $(foreach b,symmetric restart,$(WORKSPACE)/work/%/$(b)/ocean.stats) - @test "$(shell ls -A $(WORKSPACE)/results/$* 2>/dev/null)" || rm -rf $(WORKSPACE)/results/$* +.PRECIOUS: $(foreach b,symmetric restart target,$(WORK)/%/$(b)/ocean.stats) +%.restart: $(foreach b,symmetric restart,$(WORK)/%/$(b)/ocean.stats) + @test "$(shell ls -A $(WORK)/results/$* 2>/dev/null)" || rm -rf $(WORK)/results/$* @cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \ || !( \ - mkdir -p $(WORKSPACE)/results/$*; \ - (diff $^ | tee $(WORKSPACE)/results/$*/chksum_diag.restart.diff | head -n 20) ; \ + mkdir -p $(WORK)/results/$*; \ + (diff $^ | tee $(WORK)/results/$*/chksum_diag.restart.diff | head -n 20) ; \ echo -e "$(FAIL): Solutions $*.restart have changed." \ ) @echo -e "$(PASS): Solutions $*.restart agree." @@ -491,21 +492,21 @@ $(foreach c,$(CONFIGS),$(eval $(call CONFIG_DIM_RULE,$(c)))) # TODO: chksum_diag parsing of restart files # stats rule is unchanged, but we cannot use CMP_RULE to generate it. -%.regression: $(foreach b,symmetric target,$(WORKSPACE)/work/%/$(b)/ocean.stats) - @test "$(shell ls -A $(WORKSPACE)/results/$* 2>/dev/null)" || rm -rf $(WORKSPACE)/results/$* +%.regression: $(foreach b,symmetric target,$(WORK)/%/$(b)/ocean.stats) + @test "$(shell ls -A $(WORK)/results/$* 2>/dev/null)" || rm -rf $(WORK)/results/$* @cmp $^ || !( \ - mkdir -p $(WORKSPACE)/results/$*; \ - (diff $^ | tee $(WORKSPACE)/results/$*/ocean.stats.regression.diff | head -n 20) ; \ + mkdir -p $(WORK)/results/$*; \ + (diff $^ | tee $(WORK)/results/$*/ocean.stats.regression.diff | head -n 20) ; \ echo -e "$(FAIL): Solutions $*.regression have changed." \ ) @echo -e "$(PASS): Solutions $*.regression agree." # Regression testing only checks for changes in existing diagnostics -%.regression.diag: $(foreach b,symmetric target,$(WORKSPACE)/work/%/$(b)/chksum_diag) +%.regression.diag: $(foreach b,symmetric target,$(WORK)/%/$(b)/chksum_diag) @! diff $^ | grep "^[<>]" | grep "^>" > /dev/null \ || ! (\ - mkdir -p $(WORKSPACE)/results/$*; \ - (diff $^ | tee $(WORKSPACE)/results/$*/chksum_diag.regression.diff | head -n 20) ; \ + mkdir -p $(WORK)/results/$*; \ + (diff $^ | tee $(WORK)/results/$*/chksum_diag.regression.diff | head -n 20) ; \ echo -e "$(FAIL): Diagnostics $*.regression.diag have changed." \ ) @cmp $^ || ( \ @@ -534,7 +535,7 @@ tc4/configure: tc4/configure.ac #--- # Test run output files -# Rule to build $(WORKSPACE)/work//{ocean.stats,chksum_diag}. +# Rule to build $(WORK)//{ocean.stats,chksum_diag}. # $(1): Test configuration name # $(2): Executable type # $(3): Enable coverage flag @@ -543,13 +544,13 @@ tc4/configure: tc4/configure.ac # $(6): Number of MPI ranks define STAT_RULE -$(WORKSPACE)/work/%/$(1)/ocean.stats $(WORKSPACE)/work/%/$(1)/chksum_diag: build/$(2)/MOM6 | preproc +$(WORK)/%/$(1)/ocean.stats $(WORK)/%/$(1)/chksum_diag: $(BUILD)/$(2)/MOM6 | preproc @echo "Running test $$*.$(1)..." mkdir -p $$(@D) cp -RL $$*/* $$(@D) mkdir -p $$(@D)/RESTART echo -e "$(4)" > $$(@D)/MOM_override - rm -f $(WORKSPACE)/results/$$*/std.$(1).{out,err} + rm -f $(WORK)/results/$$*/std.$(1).{out,err} cd $$(@D) \ && $(TIME) $(5) $(MPIRUN) -n $(6) $(abspath $$<) 2> std.err > std.out \ || !( \ @@ -561,8 +562,8 @@ $(WORKSPACE)/work/%/$(1)/ocean.stats $(WORKSPACE)/work/%/$(1)/chksum_diag: build ) @echo -e "$(DONE): $$*.$(1); no runtime errors." if [ $(3) ]; then \ - mkdir -p $(WORKSPACE)/results/$$* ; \ - cd build/$(2) ; \ + mkdir -p $(WORK)/results/$$* ; \ + cd $(BUILD)/$(2) ; \ gcov -b *.gcda > gcov.$$*.$(1).out ; \ find -name "*.gcov" -exec sed -i -r 's/^( *[0-9]*)\*:/ \1:/g' {} \; ; \ fi @@ -585,12 +586,12 @@ codecov: .PHONY: report.cov report.cov: run.cov codecov - ./codecov $(CODECOV_TOKEN_ARG) -R build/cov -Z -f "*.gcov" \ - > build/cov/codecov.out \ - 2> build/cov/codecov.err \ + ./codecov $(CODECOV_TOKEN_ARG) -R $(BUILD)/cov -Z -f "*.gcov" \ + > $(BUILD)/cov/codecov.out \ + 2> $(BUILD)/cov/codecov.err \ && echo -e "${MAGENTA}Report uploaded to codecov.${RESET}" \ || { \ - cat build/cov/codecov.err ; \ + cat $(BUILD)/cov/codecov.err ; \ echo -e "${RED}Failed to upload report.${RESET}" ; \ if [ "$(REQUIRE_COVERAGE_UPLOAD)" = true ] ; then false ; fi ; \ } @@ -620,7 +621,7 @@ $(eval $(call STAT_RULE,cov,cov,true,,,1)) # 2. Convert DAYMAX from TIMEUNIT to seconds # 3. Apply seconds to `ocean_solo_nml` inside input.nml. # NOTE: Assumes that runtime set by DAYMAX, will fail if set by input.nml -$(WORKSPACE)/work/%/restart/ocean.stats: build/symmetric/MOM6 | preproc +$(WORK)/%/restart/ocean.stats: $(BUILD)/symmetric/MOM6 | preproc rm -rf $(@D) mkdir -p $(@D) cp -RL $*/* $(@D) @@ -634,7 +635,7 @@ $(WORKSPACE)/work/%/restart/ocean.stats: build/symmetric/MOM6 | preproc && halfperiod=$$(awk -v t=$${daymax} -v dt=$${timeunit} 'BEGIN {printf "%.f", 0.5*t*dt}') \ && printf "\n&ocean_solo_nml\n seconds = $${halfperiod}\n/\n" >> input.nml # Remove any previous archived output - rm -f $(WORKSPACE)/results/$*/std.restart{1,2}.{out,err} + rm -f $(WORK)/results/$*/std.restart{1,2}.{out,err} # Run the first half-period cd $(@D) && $(TIME) $(MPIRUN) -n 1 $(abspath $<) 2> std1.err > std1.out \ || !( \ @@ -660,7 +661,7 @@ $(WORKSPACE)/work/%/restart/ocean.stats: build/symmetric/MOM6 | preproc # Not a true rule; only call this after `make test` to summarize test results. .PHONY: test.summary test.summary: - @./tools/report_test_results.sh $(WORKSPACE)/results + @./tools/report_test_results.sh $(WORK)/results #--- @@ -668,48 +669,48 @@ test.summary: # NOTE: Using file parser gcov report as a proxy for test completion .PHONY: run.cov.unit -run.cov.unit: build/unit/MOM_file_parser_tests.F90.gcov +run.cov.unit: $(BUILD)/unit/MOM_file_parser_tests.F90.gcov .PHONY: build.unit -build.unit: $(foreach f, $(UNIT_EXECS), build/unit/$(f)) +build.unit: $(foreach f, $(UNIT_EXECS), $(BUILD)/unit/$(f)) .PHONY: run.unit run.unit: $(foreach f, $(UNIT_EXECS), work/unit/$(f).out) .PHONY: build.timing -build.timing: $(foreach f, $(TIMING_EXECS), build/timing/$(f)) +build.timing: $(foreach f, $(TIMING_EXECS), $(BUILD)/timing/$(f)) .PHONY: run.timing run.timing: $(foreach f, $(TIMING_EXECS), work/timing/$(f).out) .PHONY: show.timing show.timing: $(foreach f, $(TIMING_EXECS), work/timing/$(f).show) -$(WORKSPACE)/work/timing/%.show: +$(WORK)/timing/%.show: ./tools/disp_timing.py $(@:.show=.out) # Invoke the above unit/timing rules for a "target" code # Invoke with appropriate macros defines, i.e. -# make build.timing_target MOM_TARGET_URL=... MOM_TARGET_BRANCH=... TARGET_CODEBASE=build/target_codebase -# make run.timing_target TARGET_CODEBASE=build/target_codebase +# make build.timing_target MOM_TARGET_URL=... MOM_TARGET_BRANCH=... TARGET_CODEBASE=$(BUILD)/target_codebase +# make run.timing_target TARGET_CODEBASE=$(BUILD)/target_codebase TIMING_TARGET_EXECS ?= $(basename $(notdir $(wildcard $(TARGET_CODEBASE)/config_src/drivers/timing_tests/*.F90) ) ) .PHONY: build.timing_target -build.timing_target: $(foreach f, $(TIMING_TARGET_EXECS), $(TARGET_CODEBASE)/.testing/build/timing/$(f)) +build.timing_target: $(foreach f, $(TIMING_TARGET_EXECS), $(TARGET_CODEBASE)/.testing/$(BUILD)/timing/$(f)) .PHONY: run.timing_target run.timing_target: $(foreach f, $(TIMING_TARGET_EXECS), $(TARGET_CODEBASE)/.testing/work/timing/$(f).out) .PHONY: compare.timing compare.timing: $(foreach f, $(filter $(TIMING_EXECS),$(TIMING_TARGET_EXECS)), work/timing/$(f).compare) -$(WORKSPACE)/work/timing/%.compare: $(TARGET_CODEBASE) +$(WORK)/timing/%.compare: $(TARGET_CODEBASE) ./tools/disp_timing.py -r $(TARGET_CODEBASE)/.testing/$(@:.compare=.out) $(@:.compare=.out) $(TARGET_CODEBASE)/.testing/%: | $(TARGET_CODEBASE) cd $(TARGET_CODEBASE)/.testing && make $* # General rule to run a unit test executable -# Pattern is to run build/unit/executable and direct output to executable.out -$(WORKSPACE)/work/unit/%.out: build/unit/% +# Pattern is to run $(BUILD)/unit/executable and direct output to executable.out +$(WORK)/unit/%.out: $(BUILD)/unit/% @mkdir -p $(@D) cd $(@D) ; $(TIME) $(MPIRUN) -n 1 $(abspath $<) 2> >(tee $*.err) > $*.out -$(WORKSPACE)/work/unit/test_MOM_file_parser.out: build/unit/test_MOM_file_parser +$(WORK)/unit/test_MOM_file_parser.out: $(BUILD)/unit/test_MOM_file_parser if [ $(REPORT_COVERAGE) ]; then \ - find build/unit -name *.gcda -exec rm -f '{}' \; ; \ + find $(BUILD)/unit -name *.gcda -exec rm -f '{}' \; ; \ fi mkdir -p $(@D) cd $(@D) \ @@ -727,25 +728,25 @@ $(WORKSPACE)/work/unit/test_MOM_file_parser.out: build/unit/test_MOM_file_parser ) # NOTE: .gcov actually depends on .gcda, but .gcda is produced with std.out -# TODO: Replace $(WORKSPACE)/work/unit/std.out with *.gcda? -build/unit/MOM_file_parser_tests.F90.gcov: $(WORKSPACE)/work/unit/test_MOM_file_parser.out +# TODO: Replace $(WORK)/unit/std.out with *.gcda? +$(BUILD)/unit/MOM_file_parser_tests.F90.gcov: $(WORK)/unit/test_MOM_file_parser.out cd $(@D) \ && gcov -b *.gcda > gcov.unit.out find $(@D) -name "*.gcov" -exec sed -i -r 's/^( *[0-9]*)\*:/ \1:/g' {} \; .PHONY: report.cov.unit -report.cov.unit: build/unit/MOM_file_parser_tests.F90.gcov codecov - ./codecov $(CODECOV_TOKEN_ARG) -R build/unit -f "*.gcov" -Z -n "Unit tests" \ - > build/unit/codecov.out \ - 2> build/unit/codecov.err \ +report.cov.unit: $(BUILD)/unit/MOM_file_parser_tests.F90.gcov codecov + ./codecov $(CODECOV_TOKEN_ARG) -R $(BUILD)/unit -f "*.gcov" -Z -n "Unit tests" \ + > $(BUILD)/unit/codecov.out \ + 2> $(BUILD)/unit/codecov.err \ && echo -e "${MAGENTA}Report uploaded to codecov.${RESET}" \ || { \ - cat build/unit/codecov.err ; \ + cat $(BUILD)/unit/codecov.err ; \ echo -e "${RED}Failed to upload report.${RESET}" ; \ if [ "$(REQUIRE_COVERAGE_UPLOAD)" = true ] ; then false ; fi ; \ } -$(WORKSPACE)/work/timing/%.out: build/timing/% FORCE +$(WORK)/timing/%.out: $(BUILD)/timing/% FORCE @mkdir -p $(@D) @echo Running $< in $(@D) @cd $(@D) ; $(TIME) $(MPIRUN) -n 1 $(abspath $<) 2> $*.err > $*.out @@ -759,17 +760,17 @@ PCONFIGS = p0 profile: $(foreach p,$(PCONFIGS), prof.$(p)) .PHONY: prof.p0 -prof.p0: $(WORKSPACE)/work/p0/opt/clocks.json $(WORKSPACE)/work/p0/opt_target/clocks.json +prof.p0: $(WORK)/p0/opt/clocks.json $(WORK)/p0/opt_target/clocks.json python tools/compare_clocks.py $^ -$(WORKSPACE)/work/p0/%/clocks.json: $(WORKSPACE)/work/p0/%/std.out +$(WORK)/p0/%/clocks.json: $(WORK)/p0/%/std.out python tools/parse_fms_clocks.py -d $(@D) $^ > $@ \ || !( rm $@ ) -$(WORKSPACE)/work/p0/opt/std.out: build/opt/MOM6 -$(WORKSPACE)/work/p0/opt_target/std.out: build/opt_target/MOM6 +$(WORK)/p0/opt/std.out: $(BUILD)/opt/MOM6 +$(WORK)/p0/opt_target/std.out: $(BUILD)/opt_target/MOM6 -$(WORKSPACE)/work/p0/%/std.out: +$(WORK)/p0/%/std.out: mkdir -p $(@D) cp -RL p0/* $(@D) mkdir -p $(@D)/RESTART @@ -788,16 +789,16 @@ PERF_EVENTS ?= perf: $(foreach p,$(PCONFIGS), perf.$(p)) .PHONY: prof.p0 -perf.p0: $(WORKSPACE)/work/p0/opt/profile.json $(WORKSPACE)/work/p0/opt_target/profile.json +perf.p0: $(WORK)/p0/opt/profile.json $(WORK)/p0/opt_target/profile.json python tools/compare_perf.py $^ -$(WORKSPACE)/work/p0/%/profile.json: $(WORKSPACE)/work/p0/%/perf.data +$(WORK)/p0/%/profile.json: $(WORK)/p0/%/perf.data python tools/parse_perf.py -f $< > $@ -$(WORKSPACE)/work/p0/opt/perf.data: build/opt/MOM6 -$(WORKSPACE)/work/p0/opt_target/perf.data: build/opt_target/MOM6 +$(WORK)/p0/opt/perf.data: $(BUILD)/opt/MOM6 +$(WORK)/p0/opt_target/perf.data: $(BUILD)/opt_target/MOM6 -$(WORKSPACE)/work/p0/%/perf.data: +$(WORK)/p0/%/perf.data: mkdir -p $(@D) cp -RL p0/* $(@D) mkdir -p $(@D)/RESTART @@ -816,7 +817,7 @@ $(WORKSPACE)/work/p0/%/perf.data: .PHONY: clean clean: clean.build clean.stats @[ $$(basename $$(pwd)) = .testing ] - rm -rf deps + rm -rf $(DEPS) .PHONY: clean.build @@ -828,7 +829,7 @@ clean.build: .PHONY: clean.stats clean.stats: @[ $$(basename $$(pwd)) = .testing ] - rm -rf $(WORKSPACE)/work $(WORKSPACE)/results + rm -rf $(WORK) .PHONY: clean.preproc From 7245426edca7d7bf6f91d83060414bbd5c291a4a Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Tue, 16 Jan 2024 17:01:36 -0500 Subject: [PATCH 2/3] Use ifdef testing of DO_* flags Refactor the various DO_* flags to use ifdef, which checks if the flags have any value, rather than explicitly looking for "true". This follows the Make convention of an unset value corresponding to False. There is also some minor refactoring, such as pruning of redundant comments. --- .testing/Makefile | 56 ++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/.testing/Makefile b/.testing/Makefile index 400081fd9e..e3f553b99a 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -67,11 +67,8 @@ # TODO: POSIX shell compatibility SHELL = bash -# No implicit rules -.SUFFIXES: - -# No implicit variables -MAKEFLAGS += -R +# No implicit rules, suffixes, or variables +MAKEFLAGS += -rR # User-defined configuration -include config.mk @@ -106,10 +103,7 @@ FCFLAGS_FMS ?= $(FCFLAGS_DEBUG) LDFLAGS_COVERAGE ?= --coverage LDFLAGS_USER ?= -# Set to `true` to require identical results from DEBUG and REPRO builds -# NOTE: Many compilers (Intel, GCC on ARM64) do not produce identical results -# across DEBUG and REPRO builds (as defined below), so we disable on -# default. +# Set to verify identical DEBUG and REPRO results DO_REPRO_TESTS ?= # Enable profiling @@ -118,9 +112,12 @@ DO_PROFILE ?= # Enable code coverage runs DO_COVERAGE ?= -# Enable code coverage runs +# Enable unit tests DO_UNIT_TESTS ?= +# Check for regressions with target branch +DO_REGRESSION_TESTS ?= + # Report failure if coverage report is not uploaded REQUIRE_COVERAGE_UPLOAD ?= @@ -141,41 +138,50 @@ BUILD ?= build DEPS ?= $(BUILD)/deps WORK ?= work +# Unit test executables +UNIT_EXECS ?= \ + $(basename $(notdir $(wildcard ../config_src/drivers/unit_tests/*.F90))) + +# Timing test executables +TIMING_EXECS ?= \ + $(basename $(notdir $(wildcard ../config_src/drivers/timing_tests/*.F90))) + #--- # Test configuration -# REPRO tests enable reproducibility with optimization, and often do not match -# the DEBUG results in older GCCs and vendor compilers, so we can optionally -# disable them. -ifeq ($(DO_REPRO_TESTS), true) +# Set if either DO_COVERAGE or DO_UNIT_TESTS is set +run_unit_tests = + +# REPRO and DEBUG equivalence +ifdef DO_REPRO_TESTS EXECS += repro/MOM6 TESTS += repro endif # Profiling -ifeq ($(DO_PROFILE), true) +ifdef DO_PROFILE EXECS += opt/MOM6 opt_target/MOM6 endif # Coverage -ifeq ($(DO_COVERAGE), true) +ifdef DO_COVERAGE EXECS += cov/MOM6 + run_unit_execs = yes endif -# Unit testing (or coverage) -UNIT_EXECS ?= $(basename $(notdir $(wildcard ../config_src/drivers/unit_tests/*.F90) ) ) -TIMING_EXECS ?= $(basename $(notdir $(wildcard ../config_src/drivers/timing_tests/*.F90) ) ) -ifneq (X$(DO_COVERAGE)$(DO_UNIT_TESTS)X, XX) - BUILDS += $(foreach e, $(UNIT_EXECS), unit/$(e)) +# Unit test executables +ifdef DO_UNIT_TESTS + run_unit_tests = yes endif -ifeq ($(DO_PROFILE), false) - EXECS += opt/MOM6 opt_target/MOM6 +# If either coverage or unit tests are enabled, build the unit test execs +ifdef run_unit_tests + EXECS += $(foreach e, $(UNIT_EXECS), unit/$(e)) endif -DO_REGRESSION_TESTS ?= -ifeq ($(DO_REGRESSION_TESTS), true) +# Regression testing +ifdef DO_REGRESSION_TESTS EXECS += target/MOM6 TESTS += regression From 2095b8e4dd23aef7299584758df625458a9391eb Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Thu, 1 Feb 2024 11:05:32 -0500 Subject: [PATCH 3/3] Testing: Remove mkmf macros for coupled tests There appears to be a legacy macro and comments related to building the coupled driver object files with mkmf. To the best of my knowledge, this is no longer used, and it appears to build correctly when removed, so I am removing all reference to mkmf from this section. However, I will pack it into a separate commit, in case it is actually needed somewhere ^_^. --- .testing/Makefile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.testing/Makefile b/.testing/Makefile index e3f553b99a..98b1dee2c5 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -362,15 +362,8 @@ $(DEPS): mkdir -p $(DEPS) #--- -# The following block does a non-library build of a coupled driver interface to -# MOM, along with everything below it. This simply checks that we have not -# broken the ability to compile. This is not a means to build a complete -# coupled executable. -# TODO: -# - Avoid re-building FMS and MOM6 src by re-using existing object/mod files -# - Use autoconf rather than mkmf templates -MK_TEMPLATE ?= ../../$(DEPS)/mkmf/templates/ncrc-gnu.mk -# XXX: Are we even using this? +# Verify that the coupled model drivers can be compiled. This does not verify +# that they can be run, since it would require external submodels. # NUOPC driver $(BUILD)/nuopc/mom_ocean_model_nuopc.o: $(BUILD)/nuopc/Makefile