From c1aa968d216d006b24e9632da0cb68d6ffa070b0 Mon Sep 17 00:00:00 2001 From: Paddy Mccarthy Date: Tue, 7 Mar 2023 10:47:00 -0700 Subject: [PATCH 1/3] Fixed issue #649 and tested on cheyenne. --- scripts/exregional_aqm_lbcs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh index 70489cac7e..832d61fd0b 100755 --- a/scripts/exregional_aqm_lbcs.sh +++ b/scripts/exregional_aqm_lbcs.sh @@ -215,7 +215,7 @@ Please ensure that you've built this executable." #---------------------------------------------------------------------- # PREP_STEP - eval ${RUN_CMD_UTILS} -n ${NUMTS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ + eval ${RUN_CMD_UTILS} ${exec_fp} -n ${NUMTS} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate chemical and GEFS LBCs file for RRFS-CMAQ failed: From fa378e7c5a18b2293265d4e375084e4fe84741d6 Mon Sep 17 00:00:00 2001 From: Paddy Mccarthy Date: Tue, 7 Mar 2023 12:58:50 -0700 Subject: [PATCH 2/3] Addressed Chan-Hoo's suggestion regarding the best way to use mpirun -np ${NUMTS} on cheyenne mpiexec -n ${NUMTS} on wcoss2 srun --export=ALL -n ${NUMTS} on hera Broke these out into RUN_CMD_AQMLBC, defined in the machine/machine.yaml files. --- scripts/exregional_aqm_lbcs.sh | 2 +- ush/machine/cheyenne.yaml | 1 + ush/machine/hera.yaml | 1 + ush/machine/wcoss2.yaml | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh index 832d61fd0b..79dbd1663f 100755 --- a/scripts/exregional_aqm_lbcs.sh +++ b/scripts/exregional_aqm_lbcs.sh @@ -215,7 +215,7 @@ Please ensure that you've built this executable." #---------------------------------------------------------------------- # PREP_STEP - eval ${RUN_CMD_UTILS} ${exec_fp} -n ${NUMTS} ${REDIRECT_OUT_ERR} || \ + eval ${RUN_CMD_AQMLBC} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate chemical and GEFS LBCs file for RRFS-CMAQ failed: diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index ed10c964d8..c6b954e222 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -16,6 +16,7 @@ platform: RUN_CMD_POST: mpirun -np $nprocs RUN_CMD_SERIAL: time RUN_CMD_UTILS: mpirun -np $nprocs + RUN_CMD_AQMLBC: mpirun -np ${NUMTS} PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/input_model_data TEST_PREGEN_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/FV3LAM_pregen diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index c1b0dfe746..0a5ca74d6f 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -17,6 +17,7 @@ platform: QUEUE_HPSS: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL + RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL RUN_CMD_AQM: srun -n ${nprocs} --export=ALL diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index ad4ee2ae95..be3afa4f22 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -17,6 +17,7 @@ platform: RUN_CMD_SERIAL: mpiexec RUN_CMD_UTILS: mpiexec -n ${nprocs} RUN_CMD_AQM: mpiexec -n ${nprocs} -ppn ${ppn_run_aqm} --cpu-bind core -depth ${omp_num_threads_run_aqm} + RUN_CMD_AQMLBC: mpiexec -n ${NUMTS} SCHED_NATIVE_CMD: -l place=excl PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/input_model_data From 6369dd4f5a839d2198a70e7a672b2937839a2c52 Mon Sep 17 00:00:00 2001 From: Paddy Mccarthy Date: Wed, 8 Mar 2023 06:43:27 -0700 Subject: [PATCH 3/3] Implemented final changes suggested by Chan-Hoo. --- ush/config_defaults.yaml | 4 ++++ ush/machine/cheyenne.yaml | 1 + ush/machine/hera.yaml | 2 +- ush/machine/orion.yaml | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index fbb59040a7..fb44af7170 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -199,6 +199,9 @@ platform: # RUN_CMD_AQM: # The run command for some AQM tasks. # + # RUN_CMD_AQMLBC: + # The run command for the AQM_LBCS task. + # #----------------------------------------------------------------------- # RUN_CMD_SERIAL: "" @@ -206,6 +209,7 @@ platform: RUN_CMD_FCST: "" RUN_CMD_POST: "" RUN_CMD_AQM: "" + RUN_CMD_AQMLBC: "" # #----------------------------------------------------------------------- diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index c6b954e222..672a8043e8 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -16,6 +16,7 @@ platform: RUN_CMD_POST: mpirun -np $nprocs RUN_CMD_SERIAL: time RUN_CMD_UTILS: mpirun -np $nprocs + RUN_CMD_AQM: mpirun -np $nprocs RUN_CMD_AQMLBC: mpirun -np ${NUMTS} PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/input_model_data diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 0a5ca74d6f..08bc7a49fd 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -17,10 +17,10 @@ platform: QUEUE_HPSS: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL - RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL RUN_CMD_AQM: srun -n ${nprocs} --export=ALL + RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} SCHED_NATIVE_CMD: --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/input_model_data diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index cd9e898c35..8f03a376e8 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -20,6 +20,7 @@ platform: RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL RUN_CMD_AQM: srun --export=ALL + RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} SCHED_NATIVE_CMD: --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/input_model_data