diff --git a/.gitignore b/.gitignore index 09e62c68a5..e43342c975 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ wrfout_d* rsl.out.* rsl.error.* *.log + +# Test generated files +__pycache__/ +*.pkl diff --git a/.sane/wrf/custom_actions/run_wrf.py b/.sane/wrf/custom_actions/run_wrf.py new file mode 100644 index 0000000000..5aefe6c30d --- /dev/null +++ b/.sane/wrf/custom_actions/run_wrf.py @@ -0,0 +1,305 @@ +import os +import itertools +import shutil + +import sane + + +class WRFBase( sane.Action ): + def __init__( self, id ): + super().__init__( id ) + self.wrf_case = None + self.wrf_case_path = "${{ host_info.config.run_wrf_case_path }}" + self.wrf_dir = "test/em_real" + self.wrf_run_dir = "./output/${{ wrf_case }}" + + self.modify_environ = False + + self.wrf_exec = None + self.wrf_nml = "namelist.input" + + # Control execution + self.mpi_cmd = "mpirun -np ${{ mpi_ranks }}" + self.use_mpi = True + self.use_omp = False + self.mpi_ranks = "${{ resources.cpus }}" + self.omp_threads = "${{ resources.cpus }}" + + # Make sure we can pass on all this info + self.outputs["wrf_case"] = "${{ wrf_case }}" + self.outputs["wrf_case_path"] = "${{ wrf_case_path }}" + self.outputs["wrf_dir"] = "${{ wrf_dir }}" + self.outputs["wrf_run_dir"] = "${{ wrf_run_dir }}" + self.outputs["modify_environ"] = "${{ modify_environ }}" + self.outputs["wrf_exec"] = "${{ wrf_exec }}" + self.outputs["wrf_nml"] = "${{ wrf_nml }}" + self.outputs["mpi_cmd"] = "${{ mpi_cmd }}" + self.outputs["use_mpi"] = "${{ use_mpi }}" + self.outputs["use_omp"] = "${{ use_omp }}" + self.outputs["mpi_ranks"] = "${{ mpi_ranks }}" + self.outputs["omp_threads"] = "${{ omp_threads }}" + + def load_extra_options( self, options, origin ): + self.wrf_case = options.pop( "wrf_case", None ) + self.wrf_case_path = options.pop( "wrf_case_path", self.wrf_case_path ) + self.wrf_run_dir = options.pop( "wrf_run_dir", self.wrf_run_dir ) + self.wrf_dir = options.pop( "wrf_dir", self.wrf_dir ) + + # Do not check for execs existing yet as those may be created by other actions + self.wrf_exec = options.pop( "wrf_exec", self.wrf_exec ) + self.wrf_nml = options.pop( "wrf_nml", self.wrf_nml ) + + self.mpi_cmd = options.pop( "mpi_cmd", self.mpi_cmd ) + self.use_mpi = options.pop( "use_mpi", self.use_mpi ) + self.use_omp = options.pop( "use_omp", self.use_omp ) + self.mpi_ranks = options.pop( "mpi_ranks", self.mpi_ranks ) + self.omp_threads = options.pop( "omp_threads", self.omp_threads ) + + self.modify_environ = options.pop( "modify_environ", self.modify_environ ) + super().load_extra_options( options, origin ) + + def pre_launch( self ): + # Preflight checks + self.push_logscope( "pre_launch" ) + + # case path and case exist, force assignment check + self.wrf_case_path = self.resolve_path_exists( self.dereference( self.wrf_case_path ) ) + self.wrf_case = self.dereference( self.wrf_case ) + + if self.wrf_case is None: + msg = "No case provided" + self.log( msg, level=50 ) + raise ValueError( msg ) + # case nml exist + full_case_path = self.resolve_path_exists( os.path.join( self.wrf_case_path, self.wrf_case ) ) + self.wrf_nml = self.dereference( self.wrf_nml ) + self.file_exists_in_path( full_case_path, self.wrf_nml ) + + self.wrf_nml = self.dereference( self.wrf_nml ) + + if self.use_mpi: + self.log( f"Adding MPI command to arguments for wrf : '{self.mpi_cmd}'" ) + self.config["arguments"].extend( [ "-p", self.mpi_cmd ] ) + if self.use_omp: + self.log( f"Adding OMP_NUM_THREADS count to arguments : '{self.omp_threads}'" ) + self.config["arguments"].extend( [ "-o", self.omp_threads ] ) + + if self.use_mpi and self.use_omp: + if self.mpi_ranks == self.omp_threads and self.mpi_ranks == "${{ resources.cpus }}": + msg = "Directly set the MPI ranks and ompthreads for this action instead of default '${{ resources.cpus }}'" + self.log( msg, level=40 ) + raise Exception( msg ) + + self.pop_logscope() + + def pre_run( self ): + # Now check for things that should be here for sure since any dependencies would be + # finished by now + self.push_logscope( "pre_run" ) + full_case_path = self.resolve_path_exists( os.path.join( self.wrf_case_path, self.wrf_case ) ) + + # build location exists + self.wrf_dir = self.dereference( self.wrf_dir ) + self.wrf_dir = self.resolve_path_exists( self.wrf_dir ) + + # execs exist + self.file_exists_in_path( self.wrf_dir, self.wrf_exec ) + + if self.modify_environ: + self.log( "Adding to LD_LIBRARY_PATH..." ) + ld_lib = os.environ.get( "LD_LIBRARY_PATH", "" ) + ld_lib += f':{os.environ["NETCDF"]}/lib:{os.environ["NETCDF"]}/lib64' + os.environ["LD_LIBRARY_PATH"] = ld_lib + + self.pop_logscope() + + def setup_dir( self ): + # OK! Create run dir + self.wrf_run_dir = self.dereference( self.wrf_run_dir ) + self.wrf_run_dir = self.resolve_path( self.working_directory, self.wrf_run_dir ) + if os.path.isdir( self.wrf_run_dir ): + self.log( f"Cleaning '{self.wrf_run_dir}'" ) + shutil.rmtree( self.wrf_run_dir ) + os.makedirs( self.wrf_run_dir, exist_ok=True ) + + def setup_wrf( self ): + full_case_path = self.resolve_path_exists( os.path.join( self.wrf_case_path, self.wrf_case ) ) + + # copy over execs, then metfiles, then case to run dir + prev_exec_raw = self.__exec_raw__ + self.__exec_raw__ = False + # This should work as everything should be absolute paths + self.log( "Linking WRF executables..." ) + self.execute_subprocess( "ln", [ "-svf", os.path.join( self.wrf_dir, "*" ), self.wrf_run_dir ], verbose=True, shell=True ) + + self.log( "Copying WRF case files..." ) + self.execute_subprocess( "cp", [ "-v", "--remove-destination", os.path.join( full_case_path, "*" ), self.wrf_run_dir ], verbose=True, shell=True ) + self.__exec_raw__ = prev_exec_raw + + +class InitWRF( WRFBase ): + def __init__( self, id ): + super().__init__( id ) + self.wrf_exec = "real.exe" + + self.wrf_met_folder = "${{ wrf_case }}" + self.wrf_met_path = "${{ host_info.config.run_wrf_met_path }}" + + self.outputs["wrf_met_folder"] = "${{ wrf_met_folder }}" + self.outputs["wrf_met_path"] = "${{ wrf_met_path }}" + + # Not user input + self.config["command"] = ".sane/wrf/scripts/run_init.sh" + self.config["arguments"] = [ + "-f", "${{ wrf_run_dir }}", + "-r", "${{ wrf_exec }}", + "-n", "${{ wrf_nml }}" + ] + + def load_extra_options( self, options, origin ): + super().load_extra_options( options, origin ) + self.wrf_met_path = options.pop( "wrf_met_path", self.wrf_met_path ) + self.wrf_met_folder = options.pop( "wrf_met_folder", self.wrf_met_folder ) + + def pre_launch( self ): + super().pre_launch() + # met path exists + self.wrf_met_folder = self.dereference( self.wrf_met_folder ) + self.wrf_met_path = self.resolve_path_exists( self.dereference( self.wrf_met_path ) ) + full_met_path = self.resolve_path_exists( os.path.join( self.wrf_met_path, self.wrf_met_folder ) ) + + def pre_run( self ): + super().pre_run() + self.setup_dir() + self.setup_wrf() + self.setup_metfiles() + + def setup_metfiles( self ): + full_met_path = self.resolve_path_exists( os.path.join( self.wrf_met_path, self.wrf_met_folder ) ) + + # copy over execs, then metfiles, then case to run dir + prev_exec_raw = self.__exec_raw__ + self.__exec_raw__ = False + # This should work as everything should be absolute paths + self.log( "Linking WRF metfiles..." ) + self.execute_subprocess( "ln", [ "-svf", os.path.join( full_met_path, "*" ), self.wrf_run_dir ], verbose=True, shell=True ) + self.__exec_raw__ = prev_exec_raw + + +class RunWRF( WRFBase ): + def __init__( self, id ): + super().__init__( id ) + self.wrf_exec = "wrf.exe" + + # Override directly or inherit from InitWRF + self.wrf_case = None + self.wrf_case_path = None + self.wrf_dir = None + self.wrf_run_dir = None + self.modify_environ = None + self.wrf_nml = None + + # Should we setup the dir again + self._create_run_dir = True + self._inherit_dep = None + + # Not user input + self.config["command"] = ".sane/wrf/scripts/run_wrf.sh" + self.config["arguments"] = [ + "-f", "${{ wrf_run_dir }}", + "-r", "${{ wrf_exec }}", + "-n", "${{ wrf_nml }}" + ] + + def pre_launch( self ): + # If a dependency can provide us info that we are missing + for dep_name, dep_info in self.dependencies.items(): + attrs = [ + "wrf_case", + "wrf_case_path", + "wrf_dir", + "wrf_run_dir", + "modify_environ", + "wrf_nml" + ] + inherit = [] + for attr in attrs: + if getattr( self, attr ) is None and attr in dep_info["outputs"]: + setattr( self, attr, dep_info["outputs"][attr] ) + inherit.append( attr ) + self.log( f"Getting '{attr}' info from dependency '{dep_name}'" ) + + if "wrf_run_dir" in inherit: + self._create_run_dir = False + + if len( inherit ) > 0: + self._inherit_dep = dep_name + # Only ever inherit from one + break + super().pre_launch() + + def pre_run( self ): + super().pre_run() + if self._create_run_dir: + self.setup_dir() + self.setup_wrf() + + # Copy files needed from dep + self.setup_input() + + def setup_input( self ): + # copy over input files + full_init_path = self.resolve_path_exists( self.dependencies[self._inherit_dep]["outputs"]["wrf_run_dir"] ) + prev_exec_raw = self.__exec_raw__ + self.__exec_raw__ = False + # This should work as everything should be absolute paths + self.log( "Linking input files..." ) + self.execute_subprocess( "ln", [ "-svf", os.path.join( full_init_path, "wrf*_d*" ), self.wrf_run_dir ], verbose=True, shell=True ) + self.__exec_raw__ = prev_exec_raw + + +class RunWRFRestart( RunWRF ): + def __init__( self, id ): + super().__init__( id ) + self.wrf_restart_nml = "namelist.input.restart" + self.wrf_diff_exec = "./external/io_netcdf/diffwrf" + self.hist_comparisons = 1 + + def load_extra_options( self, options, origin ): + self.wrf_restart_nml = options.pop( "wrf_restart_nml", self.wrf_restart_nml ) + self.hist_comparisons = options.pop( "hist_comparisons", self.hist_comparisons ) + super().load_extra_options( options, origin ) + + def pre_launch( self ): + super().pre_launch( ) + if self.wrf_restart_nml is None: + raise ValueError( "No restart namelist specified" ) + + full_case_path = self.resolve_path_exists( os.path.join( self.wrf_case_path, self.wrf_case ) ) + self.wrf_restart_nml = self.dereference( self.wrf_restart_nml ) + self.file_exists_in_path( full_case_path, self.wrf_restart_nml ) + + self.wrf_diff_exec = self.dereference( self.wrf_diff_exec ) + + def run( self ): + retval = super().run() + if retval != 0: + return retval + + arg_dict = { + self.config["arguments"][i] : self.config["arguments"][i+1] + for i in range( 0, len(self.config["arguments"]), 2 ) + } + arg_dict["-n"] = self.wrf_restart_nml + arg_dict["-d"] = self.wrf_diff_exec + arg_dict["-t"] = self.hist_comparisons + self.config["command"] = ".sane/scripts/run_wrf_restart.sh" + self.config["arguments"] = list( itertools.chain( *zip( arg_dict.keys(), arg_dict.values() ) ) ) + + self.push_logscope( "run" ) + self.log( "Running restart namelist now" ) + self.pop_logscope() + + # Do it again :) + retval = super().run() + return retval diff --git a/.sane/wrf/hosts/derecho.jsonc b/.sane/wrf/hosts/derecho.jsonc new file mode 100644 index 0000000000..06027ede51 --- /dev/null +++ b/.sane/wrf/hosts/derecho.jsonc @@ -0,0 +1,65 @@ +{ + "hosts" : + { + "derecho" : + { + "type" : "PBSHost", + "resources" : + { + "cpudev" : + { + "nodes" : 1, + "exclusive" : false, + "queues" : [ "develop" ], + "resources" : { "cpus" : 256, "memory" : "235gb" } + }, + "cpu" : + { + "nodes" : 20, + "exclusive" : true, + "queues" : [ "main" ], + "resources" : { "cpus" : 128, "memory" : "256gb" } + } + }, + "local_resources" : + { + // prevent thrashing of the login nodes + "cpus" : 12 + }, + "base_env" : + { + "lmod_path" : "/glade/u/apps/derecho/24.12/spack/opt/spack/lmod/8.7.37/gcc/12.4.0/nr3e/lmod/lmod/init/env_modules_python.py", + "env_scripts" : [ "/etc/profile.d/z00_modules.sh" ] + }, + "mapping" : + { + "ncpus" : [ "cpu", "cpus", "ncpu" ] + }, + // "queue" : "main", + "account" : "NMMM0012", + "config" : + { + "wrf_coop" : + { + "run_wrf_case_path" : "/glade/work/aislas/wrf-model/SCRIPTS/Namelists/weekly/", + "run_wrf_met_path" : "/glade/work/aislas/github/data/wrf/regtest/Data/" + } + } + } + }, + "patches" : + { + "actions" : + { + // CMake builds + "[build_.*]" : { "resources" : { "derecho" : { "cpus" : 16 } } }, + "[build_cmake.*gnu.*]" : { "resources" : { "derecho" : { "timelimit" : "00:10:00" } } }, + "[build_cmake.*intel.*]" : { "resources" : { "derecho" : { "timelimit" : "00:20:00" } } }, + "[build_cmake.*nvhpc.*]" : { "resources" : { "derecho" : { "timelimit" : "00:20:00" } } }, + // Make builds + "[build_make.*gnu.*]" : { "resources" : { "derecho" : { "timelimit" : "00:15:00" } } }, + "[build_make.*intel.*]" : { "resources" : { "derecho" : { "timelimit" : "00:45:00" } } }, + "[build_make.*nvhpc.*]" : { "resources" : { "derecho" : { "timelimit" : "00:45:00" } } } + } + } +} \ No newline at end of file diff --git a/.sane/wrf/hosts/derecho_envs.jsonc b/.sane/wrf/hosts/derecho_envs.jsonc new file mode 100644 index 0000000000..3969c6d3cd --- /dev/null +++ b/.sane/wrf/hosts/derecho_envs.jsonc @@ -0,0 +1,51 @@ +{ + "patches": + { + "hosts": + { + "derecho": + { + "default_env" : "gnu", + "environments" : + { + "gnu" : + { + "aliases" : [ "gcc" ], + "lmod_cmds" : + [ + { "cmd" : "purge" }, + { "cmd" : "load", "args" : [ "gcc", "cray-mpich", "netcdf", "cmake" ] } + ] + }, + "intel-oneapi" : + { + "aliases" : [ "intel-llvm" ], + "lmod_cmds" : + [ + { "cmd" : "purge" }, + { "cmd" : "load", "args" : [ "intel-oneapi", "cray-mpich", "netcdf", "cmake" ] } + ] + }, + "intel-classic" : + { + "lmod_cmds" : + [ + // Because module is particular about flag placement :) + { "cmd" : "--force", "args" : [ "purge" ] }, + { "cmd" : "load", "args" : [ "ncarenv/23.09", "intel-oneapi", "cray-mpich", "netcdf", "cmake" ] } + ] + }, + "nvhpc" : + { + "aliases" : [ "pgi" ], + "lmod_cmds" : + [ + { "cmd" : "purge" }, + { "cmd" : "load", "args" : [ "nvhpc", "cray-mpich", "netcdf", "cmake" ] } + ] + } + } + } + } + } +} diff --git a/.sane/wrf/scripts/buildCMake.sh b/.sane/wrf/scripts/buildCMake.sh new file mode 100755 index 0000000000..7340785110 --- /dev/null +++ b/.sane/wrf/scripts/buildCMake.sh @@ -0,0 +1,61 @@ +#!/bin/sh +help() +{ + echo "./buildCMake.sh [options]" + echo " -c Configuration build type, piped directly into configure" + echo " -b Build command passed into compile" + echo " -r Clean command passed into cleanCmake" + echo " -h Print this message" + echo "" +} + + +echo "Input arguments:" +echo "$*" + +while getopts c:b:r:h opt; do + case $opt in + c) + configuration="$OPTARG" + ;; + b) + buildCommand="$OPTARG" + ;; + r) + cleanCommand="$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + +# Now evaluate env vars in case it pulls from hostenv.sh +if [ ! -z "$envVars" ]; then + setenvStr "$envVars" +fi + +echo "./cleanCMake.sh -a $cleanCommand" +./cleanCMake.sh -a $cleanCommand +echo "Clean done" + +echo "./configure_new $configuration" +./configure_new $configuration +result=$? +echo "Configure done" + +if [ $result -ne 0 ]; then + echo "Failed to configure, command returned non-zero exit status" + exit 1 +fi + +echo "./compile_new $buildCommand" +./compile_new $buildCommand +result=$? +echo "Compile done" + +if [ $result -ne 0 ]; then + echo "Failed to compile, command returned non-zero exit status" + exit 1 +fi diff --git a/.sane/wrf/scripts/buildMake.sh b/.sane/wrf/scripts/buildMake.sh new file mode 100755 index 0000000000..7ec971a688 --- /dev/null +++ b/.sane/wrf/scripts/buildMake.sh @@ -0,0 +1,89 @@ +#!/bin/sh +help() +{ + echo "./build.sh [options]" + echo " -c Configuration build type, piped directly into configure" + echo " -n Configuration nesting type, piped directly into configure" + echo " -o Configuration optstring passed into configure" + echo " -b Build command passed into compile" + echo " -d Build directory, default is './'" + echo " -h Print this message" + echo "" +} + +echo "Input arguments:" +echo "$*" + +buildDirectory="./" +while getopts c:n:o:b:d:h opt; do + case $opt in + c) + configuration="$OPTARG" + ;; + n) + nesting="$OPTARG" + ;; + o) + configOpt="$OPTARG" + ;; + b) + buildCommand="$OPTARG" + ;; + d) + buildDirectory="$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + +if [ "$buildDirectory" != "./" ]; then + if [ -d "$buildDirectory" ]; then + echo "Removing $buildDirectory" + rm -rf $buildDirectory + fi + echo "Copying WRF to $buildDirectory" + mkdir -p $buildDirectory + cp arch chem clean cleanCMake.sh cmake CMakeLists.txt \ + compile compile_new confcheck \ + configure configure_new doc \ + dyn_em external frame hydro \ + inc LICENSE.txt main Makefile phys \ + README README.md Registry run share \ + test tools var wrftladj \ + $buildDirectory -ar + cd $buildDirectory +fi + +./clean -a + +echo "Compiling with option $configuration nesting=$nesting and additional flags '$configOpt'" +./configure $configOpt << EOF +$configuration +$nesting +EOF + +if [ ! -f configure.wrf ]; then + echo "Failed to configure" + exit 1 +fi + +echo "./compile $buildCommand" +./compile $buildCommand + +result=$? + +if [ $result -ne 0 ]; then + echo "Failed to compile" + exit 1 +fi + +# And a *very* special check because WRF compiles the WRF way and force-ignores all make errors +# putting the onus on US to check for things +if [ ! -x ./main/wrf.exe ]; then # There's a bunch of other execs but this is the most important and + # doing more checks to accomodate just reinforces this bad design + echo "Failed to compile" + exit 1 +fi diff --git a/.sane/wrf/scripts/compare_wrf.sh b/.sane/wrf/scripts/compare_wrf.sh new file mode 100755 index 0000000000..a4884fd159 --- /dev/null +++ b/.sane/wrf/scripts/compare_wrf.sh @@ -0,0 +1,88 @@ +#!/bin/sh +diff_exec=$1 +original=$2 +shift +shift +folders=$* + +# For make builds that rely on modifying LD_LIBRARY_PATH +if [ -n "$NETCDF" ] && [ $( ldd $diff_exec | grep "not found" -c ) -gt 0 ]; then + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NETCDF/lib/:$NETCDF/lib64 +fi + +# Make sure OMP doesn't do multiple comparisons +export OMP_NUM_THREADS=1 + +# Go to a temp directory to make sure any comparison files generated are +# just for this file +comp_dir=$( mktemp -d tmp_comp.XXXXXXXX ) +cd $comp_dir + +echo "Comparing outputs stored in $original to $folders" + +errorMsg="" +for folder in $folders; do + echo "" + echo "" + echo "" + echo "Checking $folder..." + for io_type in input out; do + echo "" + echo "" + echo "Checking $io_type..." + for dom in d01 d02 d03; do + if [ -e $original/wrf${io_type}_${dom}* ]; then + echo "" + echo "Checking $dom..." + rm -f fort.98 fort.88 + + if [ ! -e $folder/wrf${io_type}_${dom}* ]; then + # Domain does not exist in our current run but in the provided folder, that should not happen -FAIL! + currentErrorMsg="Domain $( ls $original/wrf${io_type}_${dom}* ) file $dom exists in $original but not in $folder, cannot compare results" + echo "$currentErrorMsg" + errorMsg="$errorMsg\n$currentErrorMsg" + continue + fi + + $diff_exec $original/wrf${io_type}_${dom}* $folder/wrf${io_type}_${dom}* + result=$? + + if [ $result -ne 0 ]; then + currentErrorMsg="$diff_exec failed" + echo "$currentErrorMsg" + errorMsg="$errorMsg\n$currentErrorMsg" + continue + fi + + if [ -e fort.98 ] || [ -e fort.88 ]; then + currentErrorMsg="$( ls $original/wrf${io_type}_${dom}* ) and $( ls $folder/wrf${io_type}_${dom}* ) differ" + # if [ -n "$statDiff" ] && [ $statDiff -eq 0 ]; then + # currentErrorMsg="$currentErrorMsg - but are statistically equivalent based on checksum" + # fi + echo "$currentErrorMsg" + cat fort.98 fort.88 2>/dev/null + errorMsg="$errorMsg\n$currentErrorMsg" + continue + fi + elif [ -e $folder/wrf${io_type}_${dom}* ]; then + currentErrorMsg="Domain file $( ls $folder/wrf${io_type}_${dom}* ) exists but not in $original/, is this an error?" + echo "$currentErrorMsg" + errorMsg="$errorMsg\n$currentErrorMsg" + continue + # else + # neither has it, skip + # echo "Domain file wrf${io_type}_${dom}* not generated for namelist $namelist, skipping check" + fi + done + done +done + +cd - +rm $comp_dir -r + +if [ -z "$errorMsg" ]; then + echo "All comparisons equal" +else + printf "%b" "$errorMsg" + exit 1 +fi diff --git a/.sane/wrf/scripts/run_init.sh b/.sane/wrf/scripts/run_init.sh new file mode 100755 index 0000000000..e996f43b61 --- /dev/null +++ b/.sane/wrf/scripts/run_init.sh @@ -0,0 +1,139 @@ +#!/usr/bin/sh +help() +{ + echo "./run_init.sh [options]" + echo " -f Folder to run wrf in, assuming everything is setup there already" + echo " -r Init executable for WRF front-end (ideal/real/etc.)" + echo " -o OMP_NUM_THREADS setting for sm/openmp usage" + echo " -p Parallel launch command (MPI) for executable, e.g. mpirun, mpiexec_mpt, mpiexec -np 8 --oversubscribe" + echo "if provided, these override the namelist inside the run folder" + echo " -n Namelist to run for wrf inside run folder, otherwise namelist.input" + echo "" + echo " -h Print this message" + echo "" +} + +banner() +{ + lengthBanner=$1 + shift + # https://www.shellscript.sh/examples/banner/ + printf "#%${lengthBanner}s#\n" | tr " " "=" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " " + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*" + printf "#%${lengthBanner}s#\n" | tr " " "=" +} + + +finish() +{ + banner 42 "STOP $cmd" + + # If somehow we got here, make sure we report errors + if [ ! -z "$err" ]; then + printf "%b" "$err" + exit 1 + fi +} + +echo "Input arguments:" +echo "$*" + +wrf_nml="namelist.input" +wrf_init="real.exe" +while getopts f:r:i:w:o:q:p:n:h opt; do + case $opt in + f) + run_folder="$OPTARG" + ;; + r) + wrf_init="$OPTARG" + ;; + o) + ompthreads="$OPTARG" + ;; + p) + mpi_cmd="$OPTARG" + ;; + n) + wrf_nml="$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + + +# Go to run location now - We only operate here from now on +cd $run_folder || exit $? + +wrf_init=$( realpath $( find -L $run_folder -type f -name $wrf_init | head -n 1 ) ) + +if [ ! -x "${wrf_init}" ]; then + echo "No domain preparation executable found" + exit 1 +fi + +if [ ! -z "${ompthreads}" ]; then + echo "Setting OMP_NUM_THREADS=$ompthreads" + export OMP_NUM_THREADS=$ompthreads +fi + +################################################################################ +cmd="$mpi_cmd $wrf_init $wrf_nml" +banner 42 "START $cmd" +# Clean up output of any of these runs +rm wrfinput_d* wrfbdy_d* wrfout_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf + +# Copy namelist +if [ "$wrf_nml" != "namelist.input" ]; then + echo "Setting $wrf_nml as namelist.input" + # remove old namelist.input which may be a symlink in which case this would have failed + rm namelist.input + cp $wrf_nml namelist.input || exit $? +fi + +# Run setup +echo "Running $mpi_cmd $wrf_init" + +eval "$mpi_cmd $wrf_init" & +wrf_pid=$! + +if [ -n "$mpi_cmd" ]; then + until [ $( ls ./rsl.out.* 2>/dev/null | wc -l ) -ne 0 ]; do + sleep 1 + # check if the process is done or failed + if ! kill -0 $wrf_pid >/dev/null 2>&1; then + break + fi + done + # Output the rsl. output + tail -f $( ls ./rsl.out.* | sort | head -n 1 ) --pid $wrf_pid -n 9999 +fi + +# Get exit status +wait $wrf_pid +result=$? +if [ $result -ne 0 ]; then + err="[$wrf_nml] $mpi_cmd $wrf_init failed" + finish +fi + +# Check output per domain +maxDom=$( grep max_dom namelist.input | awk '{print $3}' | tr -d ',' ) +currentDom=0 +while [ $currentDom -lt $maxDom ]; do + currentDom=$(( $currentDom + 1 )) + + domFiles=$( ls -1 | grep wrfinput_d0${currentDom} | wc -l | awk '{print $1}' ) + + if [ $domFiles -eq 0 ]; then + err="[$wrf_nml] No output files generated for domain $currentDom" + finish + fi +done + +finish diff --git a/.sane/wrf/scripts/run_wrf.sh b/.sane/wrf/scripts/run_wrf.sh new file mode 100755 index 0000000000..b1e006c3ce --- /dev/null +++ b/.sane/wrf/scripts/run_wrf.sh @@ -0,0 +1,137 @@ +#!/usr/bin/sh +help() +{ + echo "./run_wrf.sh [options]" + echo " -f Folder to run wrf in, assuming everything is setup there already" + echo " -r WRF executable for WRF, default is wrf.exe" + echo " -o OMP_NUM_THREADS setting for sm/openmp usage" + echo " -p Parallel launch command (MPI) for executable, e.g. mpirun, mpiexec_mpt, mpiexec -np 8 --oversubscribe" + echo "if provided, these override the namelist inside the run folder" + echo " -n Namelist to run for wrf inside run folder, otherwise namelist.input" + echo "" + echo " -h Print this message" + echo "" +} + +banner() +{ + lengthBanner=$1 + shift + # https://www.shellscript.sh/examples/banner/ + printf "#%${lengthBanner}s#\n" | tr " " "=" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " " + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*" + printf "#%${lengthBanner}s#\n" | tr " " "=" +} + + +finish() +{ + banner 42 "STOP $cmd" + + # If somehow we got here, make sure we report errors + if [ ! -z "$err" ]; then + printf "%b" "$err" + exit 1 + fi +} + +echo "Input arguments:" +echo "$*" + +wrf_nml="namelist.input" +wrf_exec="wrf.exe" +while getopts f:r:o:p:n:h opt; do + case $opt in + f) + run_folder="$OPTARG" + ;; + r) + wrf_exec="$OPTARG" + ;; + o) + ompthreads="$OPTARG" + ;; + p) + mpi_cmd="$OPTARG" + ;; + n) + wrf_nml="$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + + +# Go to run location now - We only operate here from now on +cd $run_folder || exit $? + +wrf_exec=$( realpath $( find -L $run_folder -type f -name $wrf_exec | head -n 1 ) ) + +if [ ! -x "${wrf_exec}" ]; then + echo "No wrf executable found" + exit 1 +fi + +if [ ! -z "${ompthreads}" ]; then + echo "Setting OMP_NUM_THREADS=$ompthreads" + export OMP_NUM_THREADS=$ompthreads +fi + +################################################################################ +cmd="$mpi_cmd $wrf_exec $wrf_nml" +banner 42 "START $cmd" + +# Copy namelist +if [ "$wrf_nml" != "namelist.input" ]; then + echo "Setting $wrf_nml as namelist.input" + # remove old namelist.input which may be a symlink in which case this would have failed + rm namelist.input + cp $wrf_nml namelist.input || exit $? +fi + +# Run setup +echo "Running $mpi_cmd $wrf_exec" + +eval "$mpi_cmd $wrf_exec" & +wrf_pid=$! + +if [ -n "$mpi_cmd" ]; then + until [ $( ls ./rsl.out.* 2>/dev/null | wc -l ) -ne 0 ]; do + sleep 1 + # check if the process is done or failed + if ! kill -0 $wrf_pid >/dev/null 2>&1; then + break + fi + done + # Output the rsl. output + tail -f $( ls ./rsl.out.* | sort | head -n 1 ) --pid $wrf_pid -n 9999 +fi + +# Get exit status +wait $wrf_pid +result=$? +if [ $result -ne 0 ]; then + err="[$wrf_nml] $mpi_cmd $wrf_exec failed" + finish +fi + +# Check output per domain +maxDom=$( grep max_dom namelist.input | awk '{print $3}' | tr -d ',' ) +currentDom=0 +while [ $currentDom -lt $maxDom ]; do + currentDom=$(( $currentDom + 1 )) + + domFiles=$( ls -1 | grep wrfout_d0${currentDom} | wc -l | awk '{print $1}' ) + + if [ $domFiles -eq 0 ]; then + err="[$wrf_nml] No output files generated for domain $currentDom" + finish + fi +done + +finish diff --git a/.sane/wrf/scripts/run_wrf_restart.sh b/.sane/wrf/scripts/run_wrf_restart.sh new file mode 100755 index 0000000000..b09a77fc70 --- /dev/null +++ b/.sane/wrf/scripts/run_wrf_restart.sh @@ -0,0 +1,181 @@ +#!/usr/bin/sh +help() +{ + echo "./run_wrf_restart.sh [options]" + echo " -f Folder to run wrf in, assuming everything is setup there already and WRF has already been run" + echo " -d Diff executable to use for comparing runs, default is ./external/io_netcdf/diffwrf" + echo " -r WRF executable for WRF, default is wrf.exe" + echo " -o OMP_NUM_THREADS setting for sm/openmp usage" + echo " -p Parallel launch command (MPI) for wrf, e.g. mpirun, mpiexec_mpt, mpiexec -np 8 --oversubscribe" + echo " -t Number of history files to compare, starting from the end of the restart" + echo "if provided, these override the namelist inside the run folder" + echo " -n Namelist to run for wrf inside run folder, otherwise namelist.input" + echo "" + echo " -h Print this message" + echo "" +} + +banner() +{ + lengthBanner=$1 + shift + # https://www.shellscript.sh/examples/banner/ + printf "#%${lengthBanner}s#\n" | tr " " "=" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " " + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*" + printf "#%${lengthBanner}s#\n" | tr " " "=" +} + + +finish() +{ + banner 42 "STOP $cmd" + + # If somehow we got here, make sure we report errors + if [ ! -z "$err" ]; then + printf "%b" "$err" + exit 1 + fi +} + +echo "Input arguments:" +echo "$*" + +wrf_nml="namelist.input" +wrf_exec="wrf.exe" +diffwrf="./external/io_netcdf/diffwrf" +hist_comparisons=1 +while getopts f:d:r:o:p:t:n:h opt; do + case $opt in + f) + run_folder="$OPTARG" + ;; + d) + diffwrf="$OPTARG" + ;; + r) + wrf_exec="$OPTARG" + ;; + o) + ompthreads="$OPTARG" + ;; + p) + mpi_cmd="$OPTARG" + ;; + t) + hist_comparisons="$OPTARG" + ;; + n) + wrf_nml="$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + +# May not be in the run dir so search first from working dir +diffwrf=$( realpath $diffwrf ) + +# Go to run location now - We only operate here from now on +cd $run_folder || exit $? + +wrf_exec=$( realpath $( find -L $run_folder -type f -name $wrf_exec | head -n 1 ) ) + +# Check our paths +if [ ! -x "${wrf_exec}" ]; then + echo "No wrf executable found" + exit 1 +fi +if [ ! -x "${diffwrf}" ]; then + echo "No diffwrf executable found" + exit 1 +fi +if [ ! -z "${ompthreads}" ]; then + echo "Setting OMP_NUM_THREADS=$ompthreads" + export OMP_NUM_THREADS=$ompthreads +fi + +################################################################################ +cmd="$mpi_cmd $wrf_exec $wrf_nml" +banner 42 "START $wrf_nml" +# Move previous output of previous run to safe spot +ls wrfout_d0* | xargs -i mv {} {}.orig +cp $( ls rsl.out.* | sort | head -n 1 ) rsl.out.orig + +# Remove previous diffing +rm *.diff_log + +# Copy namelist +if [ "$wrf_nml" != "namelist.input" ]; then + echo "Setting $wrf_nml as namelist.input" + # remove old namelist.input which may be a symlink in which case this would have failed + rm namelist.input + cp $wrf_nml namelist.input || exit $? +fi + +# run wrf +echo "Running $mpi_cmd $wrf_exec" + +eval "$mpi_cmd $wrf_exec" & +wrf_pid=$! + +if [ -n "$mpi_cmd" ]; then + until [ $( ls ./rsl.out.* 2>/dev/null | wc -l ) -ne 0 ]; do + sleep 1 + # check if the process is done or failed + if ! kill -0 $wrf_pid >/dev/null 2>&1; then + break + fi + done + # Output the rsl. output + tail -f $( ls ./rsl.out.* | sort | head -n 1 ) --pid $wrf_pid -n 9999 +fi + +wait $wrf_pid +result=$? +if [ $result -ne 0 ]; then + err="[$wrf_nml] $mpi_cmd $wrf_exec failed" + finish +fi + +# Check output per domain +maxDom=$( grep max_dom namelist.input | awk '{print $3}' | tr -d ',' ) +currentDom=0 +while [ $currentDom -lt $maxDom ]; do + currentDom=$(( $currentDom + 1 )) + + domFiles=$( ls -1 | grep wrfout_d0${currentDom} | wc -l | awk '{print $1}' ) + + if [ $domFiles -eq 0 ]; then + err="[$wrf_nml] No output files generated for domain $currentDom" + finish + fi +done + +echo "Comparing output of restart to previous run" +currentDom=0 +while [ $currentDom -lt $maxDom ]; do + currentDom=$(( $currentDom + 1 )) + # Skip first file as that doesn't match?? Gotta ask Kelly or Wei if that is expected + dom_files=$( ls -1 wrfout_d0${currentDom}* | grep -vE ".orig$" | sort | tail -n $hist_comparisons ) + for wrfout in $dom_files; do + $diffwrf $wrfout.orig $wrfout | tee $wrfout.diff_log + result=$? + if [ $result -ne 0 ]; then + err="[$wrfout] $diffwrf failed" + finish + fi + if [ -e fort.98 ] || [ -e fort.88 ]; then + cat fort.98 fort.88 2>/dev/null + err="[$wrfout] $diffwrf failed" + finish + fi + # otherwise + echo "[ok] $wrfout.orig and $wrfout match" + done +done + +finish diff --git a/.sane/wrf/tests/builds/builds.py b/.sane/wrf/tests/builds/builds.py new file mode 100644 index 0000000000..3519f27c56 --- /dev/null +++ b/.sane/wrf/tests/builds/builds.py @@ -0,0 +1,87 @@ +import sane +import itertools + +@sane.register +def add_build_for_envs_cmake( orch ): + envs = [ "gnu", "intel-classic", "intel-oneapi", "nvhpc" ] + env2opt = [ "gfortran", "ifort", "ifx", "pgf90" ] + sm_opt = [ "ON", "OFF" ] + dm_opt = [ "ON", "OFF" ] + build_types = [ "Release", "Debug" ] + configurations = { "ARW" : [ "EM_REAL", "EM_FIRE", "EM_B_WAVE" ] } + orch.log( f"Creating CMake build permutations..." ) + for core, env, build_type, sm, dm in itertools.product( configurations, envs, build_types, sm_opt, dm_opt ): + for case in configurations[core]: + sm_desc = "_sm" if sm == "ON" else "" + dm_desc = "_dm" if dm == "ON" else "" + id = f"{core}_{case}_{env}_{build_type}{sm_desc}{dm_desc}".lower() + + action = sane.Action( f"build_cmake_{id}" ) + action.config["command"] = ".sane/wrf/scripts/buildCMake.sh" + action.config["compiler"] = env2opt[envs.index(env)] + action.config["build_dir"] = "_${{ id }}" + action.config["install_dir"] = "install_${{ id }}" + action.config["build_type"] = build_type + action.config["core"] = core + action.config["case"] = case + action.config["dm"] = dm + action.config["sm"] = sm + + action.outputs["install_dir"] = action.config["install_dir"] + args = [] + + config_cmd = [ "-p ${{ config.compiler }}", "-d", "${{ config.build_dir }}", "-i", "${{ config.install_dir }}" ] + config_cmd.extend( [ "-x -- -DWRF_NESTING=BASIC", "-DCMAKE_BUILD_TYPE=${{ config.build_type }}" ] ) + config_cmd.extend( [ "-DWRF_CORE=${{ config.core }}", "-DWRF_CASE=${{ config.case }}" ] ) + config_cmd.extend( [ "-DUSE_MPI=${{ config.dm }}", "-DUSE_OPENMP=${{ config.sm }}" ] ) + build_cmd = [ "${{ config.build_dir }}", "-j ${{ resources.cpus }}" ] + clean_cmd = [ "-d", "${{ config.build_dir }}", "-i", "${{ config.install_dir }}" ] + + args.extend( [ "-c", " ".join( config_cmd ) ] ) + args.extend( [ "-b", " ".join( build_cmd ) ] ) + args.extend( [ "-r", " ".join( clean_cmd ) ] ) + + action.config["arguments"] = args + action.add_resource_requirements( { "cpus" : 8 } ) + action.environment = env + orch.add_action( action ) + + +@sane.register +def add_build_for_envs_make( orch ): + envs = [ "gnu", "intel-classic", "intel-oneapi", "nvhpc" ] + # Assumes x86 + env2opt = [ 32, 13, 76, 52 ] + + par_opt = { "serial" : 0, "smpar" : 1, "dmpar" : 2, "dm_sm" : 3 } + debug = [ True, False ] + targets = [ "em_real", "em_fire", "em_b_wave" ] + orch.log( f"Creating Make build permutations..." ) + for target, env, dbg, opt in itertools.product( targets, envs, debug, par_opt ): + build_type = "debug" if dbg else "release" + id = f"{target}_{env}_{build_type}_{opt}".lower() + + action = sane.Action( f"build_make_{id}" ) + action.config["command"] = ".sane/wrf/scripts/buildMake.sh" + action.config["compile_opt"] = env2opt[envs.index(env)] + par_opt[opt] + action.config["build_dir"] = "_${{ id }}" + action.config["target"] = target + action.config["nesting"] = 1 + action.config["par_opt"] = opt + action.config["optstr"] = "-d" if dbg else "" + action.outputs["build_dir"] = action.config["build_dir"] + args = [] + + args.extend( [ "-c", "${{ config.compile_opt }}", "-n", "${{ config.nesting }}" ] ) + if action.config["optstr"]: + args.extend( [ "-o", "${{ config.optstr }}" ] ) + + args.extend( [ "-b", "${{ config.target }} -j ${{ resources.cpus }}" ] ) + args.extend( [ "-d", "${{ config.build_dir }}" ] ) + + action.config["arguments"] = args + action.add_resource_requirements( { "cpus" : 8 } ) + action.environment = env + orch.add_action( action ) + + diff --git a/.sane/wrf/tests/regtests/wrf_coop.py b/.sane/wrf/tests/regtests/wrf_coop.py new file mode 100644 index 0000000000..84c037ffbf --- /dev/null +++ b/.sane/wrf/tests/regtests/wrf_coop.py @@ -0,0 +1,304 @@ +import copy + +import sane +from sane.helpers import recursive_update as dict_update +import wrf.custom_actions.run_wrf as run_wrf + + +@sane.register +def wrf_coop_reg_tests( orch ): + # init and run are based off of this default + default = { + "wrf_run_dir" : "regtests/output/${{ id }}", + "environment" : "gnu", + "modify_environ" : True, + } + # init settings, most of which are inherited to run + default_init = { + "wrf_case" : "${{ config.case }}/${{ config.par_opt }}", + "wrf_case_path" : "${{ host_info.config.wrf_coop.run_wrf_case_path }}", + "wrf_met_path" : "${{ host_info.config.wrf_coop.run_wrf_met_path }}", + "wrf_met_folder": "em_real", + "wrf_dir" : "${{ dependencies.${{ config.build }}.outputs.build_dir }}/test/${{ config.wrf_dir }}", + "resources" : { "cpus" : 1 }, + "config" : + { + "wrf_dir" : "${{ config.target }}" + } + } + # run settings + default_run = { + "resources" : { "cpus" : 8 }, + } + # override cummulative settings based on run mode + default_par_opt = { + "serial" : { "resources" : { "timelimit" : "00:25:00", "cpus" : 1 } }, + "openmp" : { "resources" : { "timelimit" : "00:25:00" } }, + "mpi" : { "resources" : { "timelimit" : "00:15:00" } }, + } + + ############################################################################## + ## Test cases + ## + ## These test cases are laid out to mirror the structure of the original + ## WRF Coop testing layout. The root node is the primary test case that + ## determines the parent namelist case folder. Under this dictionary + ## a list of comparisons, target wrf/test/ folder and set of namelist to test + ## are provided. The comparisons are used to determine run parameters as well + ## as the sub-folder in the parent namelist case folder. The most notable + ## customization here is that the "nml_cases" keys note the namelist. + ## while the dict value provides config overrides beyond the aggregated + ## defaults. These dicts *can* also hold sub-dicts matching the "compare" list + ## to provide comparison-specific overrides for that specific namelist. + ############################################################################## + wrf_cases = { + # Most of these need BUILD_RRTMG_FAST=1 and BUILD_RRTMK=1 + "em_real" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "3dtke" : { }, + "rap" : { }, + "conus" : { }, + "tropical" : { } + } + }, + "em_realA" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "03" : { }, + "03DF" : { } + } + }, + "em_realB" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "10" : { }, + "11" : { }, + "14" : { }, + "16" : { } + } + }, + "em_realC" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "17" : { }, + "18" : { }, + "20" : { } + } + }, + "em_realD" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "38" : { }, + "48" : { }, + "49" : { } + } + }, + "em_realE" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "52DF" : { } + } + }, + "em_realF" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "65DF" : { } + } + }, + "em_realG" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "kiaps1NE" : { "resources" : { "cpus" : 9 } }, + "kiaps2" : { } + } + }, + "em_realH" : + { + "compare" : ["serial", "mpi" ], + "target" : "em_real", + "nml_cases" : + { + "52" : { }, + "cmt" : { }, + "solaraNE" : { "resources" : { "cpus" : 9 } }, + "urb3bNE" : { "resources" : { "cpus" : 9 } }, + "03ST" : { }, + } + }, + "em_realI" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "03FD" : { }, + "06" : { } + } + }, + "em_realJ" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "50" : { }, + "51" : { } + } + }, + "em_realK" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "52FD" : { }, + "60" : { }, + "60NE" : { "resources" : { "cpus" : 9 } } + } + }, + "em_realL" : + { + "compare" : ["serial", "mpi", "openmp"], + "target" : "em_real", + "nml_cases" : + { + "66FD" : { }, + "71" : { }, + "78" : { }, + "79" : { } + } + } + } + + ############################################################################## + ## Create the Actions + ## + ## for each case: + ## for each the namelist that will be tested: + ## create an InitWRF action that will create the initial conditions for + ## this namelist only once + ## + ## for each comparison type: + ## create a RunWRF with a dependency to the above InitWRF + ## + ## if multiple comparisons: + ## create a comparison sane.Action dependent on all RunWRF for this nml + ## + ## create a final sync sane.Action that does nothing but is dependent on + ## the comparison sane.Action or all the RunWRF if the comparison DNE + ############################################################################## + for wrf_case, case_dict in wrf_cases.items(): + # Loop over all the base and A-L cases + build = f"build_make_{case_dict['target']}_gnu_debug_dm_sm" + + for nml_case, config in case_dict["nml_cases"].items(): + # Within a case, loop over the specific namelist to test + # as well as any specific config options for that nml case + nml = f"namelist.input.{nml_case}" + base_opts = copy.deepcopy( default ) + base_opts["wrf_nml"] = nml + base_opts["config"] = {} + base_opts["config"]["case"] = wrf_case + base_opts["config"]["target"] = case_dict["target"] + base_opts["config"]["build"] = build + + # Create the initial conditions for this nml + init_wrf = run_wrf.InitWRF( f"{wrf_case}_{nml_case}_init" ) + init_wrf.omp_threads = 1 + init_wrf.use_omp = True + + # Create the config dict to set the parameters for this init wrf + opts = dict_update( copy.deepcopy( base_opts ), copy.deepcopy( default_init ) ) + opts = dict_update( opts, copy.deepcopy( default_par_opt["serial"] ) ) + opts["config"]["par_opt"] = "SERIAL" + + init_wrf.add_dependencies( opts["config"]["build"] ) + init_wrf.load_options( opts ) + orch.add_action( init_wrf ) + + for comp in case_dict["compare"]: + # For each nml case, run WRF multiple times decomposing the domain using + # different methods and comparing at the end + id = f"{wrf_case}_{nml_case}_{comp}" + action = run_wrf.RunWRF( id ) + + # Get the general options for this nml case + general_opts = copy.deepcopy( config ) + # pull out the specifics + specifics = { c : general_opts.pop( c, {} ) for c in case_dict["compare"] } + + # Create the config dict + opts = dict_update( copy.deepcopy( base_opts ), copy.deepcopy( default_run ) ) + opts = dict_update( opts, copy.deepcopy( default_par_opt[comp]) ) + opts = dict_update( dict_update( opts, general_opts ), specifics[comp] ) + opts["config"]["par_opt"] = comp.upper() + + # Force psuedo-serial operation + action.use_mpi = True + action.use_omp = True + if comp == "mpi": + action.omp_threads = 1 + elif comp == "openmp": + action.mpi_ranks = 1 + else: + action.mpi_ranks = 1 + action.omp_threads = 1 + + # Capture output data + action.outputs["data"] = opts["wrf_run_dir"] + action.add_dependencies( opts["config"]["build"], init_wrf.id ) + action.load_options( opts ) + orch.add_action( action ) + + if len( case_dict["compare"] ) > 1: + # Create another action to compare all of them + id = f"{wrf_case}_{nml_case}" + action = sane.Action( id ) + action.config["command"] = ".sane/wrf/scripts/compare_wrf.sh" + action.config["arguments"] = [ + "${{ dependencies.${{ config.build }}.outputs.build_dir }}/external/io_netcdf/diffwrf", + *[ f"${{{{ dependencies.{wrf_case}_{nml_case}_{comp}.outputs.data }}}}" for comp in case_dict["compare"] ] + ] + action.config["build"] = build + action.environment = default["environment"] + # Not very intesive + action.local = True + action.add_dependencies( *[ f"{wrf_case}_{nml_case}_{comp}" for comp in case_dict["compare"] ], build ) + action.add_resource_requirements( { "cpus" : 1 } ) + orch.add_action( action ) + # Now add one final action that allows us to run this full case + action = sane.Action( f"{wrf_case}" ) + # just a sync + action.local = True + action.config["command"] = "echo" + action.config["arguments"] = [ "final sync step, nothing here" ] + if len( case_dict["compare"] ) > 1: + action.add_dependencies( *[ f"{wrf_case}_{nml_case}" for nml_case in case_dict["nml_cases"] ] ) + else: + action.add_dependencies( *[ f"{wrf_case}_{nml_case}_{comp}" for comp in case_dict["compare"] for nml_case in case_dict["nml_cases"] ] ) + orch.add_action( action )