From 2b4c88517e2b94878932cd444b4e6c9cb6b76573 Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Thu, 5 Oct 2023 12:07:16 -0800 Subject: [PATCH 1/6] +REMAP_AUX needs at least one more halo update. - This one is for CS%u_av, CS%v_av, which need to be updated coming into step_MOM_dyn_split_RK2. --- src/core/MOM_dynamics_split_RK2.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/MOM_dynamics_split_RK2.F90 b/src/core/MOM_dynamics_split_RK2.F90 index ae9e304736..68c6ff8069 100644 --- a/src/core/MOM_dynamics_split_RK2.F90 +++ b/src/core/MOM_dynamics_split_RK2.F90 @@ -1194,7 +1194,9 @@ subroutine remap_dyn_split_RK2_aux_vars(G, GV, CS, h_old, h_new, ALE_CSp, OBC, d if (CS%store_CAu) then call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%u_av, CS%v_av, OBC, dzRegrid) + call pass_vector(CS%u_av, CS%v_av, G%Domain) call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%CAu_pred, CS%CAv_pred, OBC, dzRegrid) + call pass_vector(CS%CAu_pred, CS%CAv_pred, G%Domain) endif call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%diffu, CS%diffv, OBC, dzRegrid) From f8c412655f6365e90337baf5ee67c4256d913779 Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Fri, 6 Oct 2023 17:36:52 -0800 Subject: [PATCH 2/6] +Next stab at fixing REMAP_AUX fallout. - This fixes the Bering ORLANSKI OBCs for differing processor counts. - This is either the wrong way to do group_pass for OBLIQUE OBC's or there is more wrong with them. --- src/core/MOM_dynamics_split_RK2.F90 | 4 ++-- src/core/MOM_open_boundary.F90 | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/core/MOM_dynamics_split_RK2.F90 b/src/core/MOM_dynamics_split_RK2.F90 index 68c6ff8069..455d8aa678 100644 --- a/src/core/MOM_dynamics_split_RK2.F90 +++ b/src/core/MOM_dynamics_split_RK2.F90 @@ -1194,9 +1194,9 @@ subroutine remap_dyn_split_RK2_aux_vars(G, GV, CS, h_old, h_new, ALE_CSp, OBC, d if (CS%store_CAu) then call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%u_av, CS%v_av, OBC, dzRegrid) - call pass_vector(CS%u_av, CS%v_av, G%Domain) + call pass_vector(CS%u_av, CS%v_av, G%Domain, complete=.false.) call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%CAu_pred, CS%CAv_pred, OBC, dzRegrid) - call pass_vector(CS%CAu_pred, CS%CAv_pred, G%Domain) + call pass_vector(CS%CAu_pred, CS%CAv_pred, G%Domain, complete=.true.) endif call ALE_remap_velocities(ALE_CSp, G, GV, h_old, h_new, CS%diffu, CS%diffv, OBC, dzRegrid) diff --git a/src/core/MOM_open_boundary.F90 b/src/core/MOM_open_boundary.F90 index 36a71e3d52..eb6a186ffd 100644 --- a/src/core/MOM_open_boundary.F90 +++ b/src/core/MOM_open_boundary.F90 @@ -9,6 +9,8 @@ module MOM_open_boundary use MOM_cpu_clock, only : cpu_clock_id, cpu_clock_begin, cpu_clock_end, CLOCK_ROUTINE use MOM_diag_mediator, only : diag_ctrl, time_type use MOM_domains, only : pass_var, pass_vector +use MOM_domains, only : create_group_pass, do_group_pass, group_pass_type +use MOM_domains, only : start_group_pass, complete_group_pass use MOM_domains, only : To_All, EAST_FACE, NORTH_FACE, SCALAR_PAIR, CGRID_NE, CORNER use MOM_error_handler, only : MOM_mesg, MOM_error, FATAL, WARNING, NOTE, is_root_pe use MOM_file_parser, only : get_param, log_version, param_file_type, log_param @@ -366,6 +368,7 @@ module MOM_open_boundary !! for remapping. Values below 20190101 recover the remapping !! answers from 2018, while higher values use more robust !! forms of the same remapping expressions. + type(group_pass_type) :: pass_oblique !< Structure for group halo pass end type ocean_OBC_type !> Control structure for open boundaries that read from files. @@ -1869,9 +1872,10 @@ subroutine open_boundary_init(G, GV, US, param_file, OBC, restart_CS) if (OBC%radiation_BCs_exist_globally) call pass_vector(OBC%rx_normal, OBC%ry_normal, G%Domain, & To_All+Scalar_Pair) if (OBC%oblique_BCs_exist_globally) then - call pass_vector(OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) - call pass_vector(OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) - call pass_vector(OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) + call create_group_pass(OBC%pass_oblique, OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) + call create_group_pass(OBC%pass_oblique, OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) + call create_group_pass(OBC%pass_oblique, OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) + call do_group_pass(OBC%pass_oblique, G%Domain) endif if (allocated(OBC%tres_x) .and. allocated(OBC%tres_y)) then do m=1,OBC%ntr @@ -5571,6 +5575,11 @@ subroutine remap_OBC_fields(G, GV, h_old, h_new, OBC, PCM_cell) enddo endif enddo ; endif ; endif + if (OBC%radiation_BCs_exist_globally) call pass_vector(OBC%rx_normal, OBC%ry_normal, G%Domain, & + To_All+Scalar_Pair) + if (OBC%oblique_BCs_exist_globally) then + call do_group_pass(OBC%pass_oblique, G%Domain) + endif end subroutine remap_OBC_fields From 2203219769cf02d10aa164dc3c65c03ec4e1837f Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Sat, 7 Oct 2023 12:23:53 -0800 Subject: [PATCH 3/6] Adding a group pass, still not solving the problem - Problem is in tangential_vel at tile boundaries. It matches right at the boundary, but needs some halo points to match too. --- src/core/MOM_open_boundary.F90 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/core/MOM_open_boundary.F90 b/src/core/MOM_open_boundary.F90 index eb6a186ffd..9e727c6ff7 100644 --- a/src/core/MOM_open_boundary.F90 +++ b/src/core/MOM_open_boundary.F90 @@ -10,7 +10,6 @@ module MOM_open_boundary use MOM_diag_mediator, only : diag_ctrl, time_type use MOM_domains, only : pass_var, pass_vector use MOM_domains, only : create_group_pass, do_group_pass, group_pass_type -use MOM_domains, only : start_group_pass, complete_group_pass use MOM_domains, only : To_All, EAST_FACE, NORTH_FACE, SCALAR_PAIR, CGRID_NE, CORNER use MOM_error_handler, only : MOM_mesg, MOM_error, FATAL, WARNING, NOTE, is_root_pe use MOM_file_parser, only : get_param, log_version, param_file_type, log_param @@ -1872,6 +1871,9 @@ subroutine open_boundary_init(G, GV, US, param_file, OBC, restart_CS) if (OBC%radiation_BCs_exist_globally) call pass_vector(OBC%rx_normal, OBC%ry_normal, G%Domain, & To_All+Scalar_Pair) if (OBC%oblique_BCs_exist_globally) then +! call pass_vector(OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) call create_group_pass(OBC%pass_oblique, OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) call create_group_pass(OBC%pass_oblique, OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) call create_group_pass(OBC%pass_oblique, OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) @@ -3278,6 +3280,12 @@ subroutine radiation_open_bdry_conds(OBC, u_new, u_old, v_new, v_old, G, GV, US, call open_boundary_apply_normal_flow(OBC, G, GV, u_new, v_new) call pass_vector(u_new, v_new, G%Domain, clock=id_clock_pass) + if (OBC%oblique_BCs_exist_globally) then + call do_group_pass(OBC%pass_oblique, G%Domain) +! call pass_vector(OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) + endif end subroutine radiation_open_bdry_conds @@ -5579,6 +5587,9 @@ subroutine remap_OBC_fields(G, GV, h_old, h_new, OBC, PCM_cell) To_All+Scalar_Pair) if (OBC%oblique_BCs_exist_globally) then call do_group_pass(OBC%pass_oblique, G%Domain) +! call pass_vector(OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) +! call pass_vector(OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) endif end subroutine remap_OBC_fields From 9753901a02fcbeb99b3723122bc164cbef65daca Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Sun, 8 Oct 2023 10:41:18 -0800 Subject: [PATCH 4/6] +Fixing oblique OBCs - Without this, u_av and v_av don't update a wide enough halo to get answers to reproduce across different proocessor counts with oblique OBCs. --- src/core/MOM_dynamics_split_RK2.F90 | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/core/MOM_dynamics_split_RK2.F90 b/src/core/MOM_dynamics_split_RK2.F90 index 1b7787554c..7e66fa1352 100644 --- a/src/core/MOM_dynamics_split_RK2.F90 +++ b/src/core/MOM_dynamics_split_RK2.F90 @@ -388,7 +388,7 @@ subroutine step_MOM_dyn_split_RK2(u, v, h, tv, visc, Time_local, dt, forces, p_s logical :: showCallTree, sym integer :: i, j, k, is, ie, js, je, Isq, Ieq, Jsq, Jeq, nz - integer :: cont_stencil + integer :: cont_stencil, obc_stencil is = G%isc ; ie = G%iec ; js = G%jsc ; je = G%jec ; nz = GV%ke Isq = G%IscB ; Ieq = G%IecB ; Jsq = G%JscB ; Jeq = G%JecB @@ -451,19 +451,21 @@ subroutine step_MOM_dyn_split_RK2(u, v, h, tv, visc, Time_local, dt, forces, p_s !--- begin set up for group halo pass cont_stencil = continuity_stencil(CS%continuity_CSp) + obc_stencil = 2 + if (CS%OBC%oblique_BCs_exist_globally) obc_stencil = 3 call cpu_clock_begin(id_clock_pass) call create_group_pass(CS%pass_eta, eta, G%Domain, halo=1) call create_group_pass(CS%pass_visc_rem, CS%visc_rem_u, CS%visc_rem_v, G%Domain, & To_All+SCALAR_PAIR, CGRID_NE, halo=max(1,cont_stencil)) call create_group_pass(CS%pass_uvp, up, vp, G%Domain, halo=max(1,cont_stencil)) call create_group_pass(CS%pass_hp_uv, hp, G%Domain, halo=2) - call create_group_pass(CS%pass_hp_uv, u_av, v_av, G%Domain, halo=2) - call create_group_pass(CS%pass_hp_uv, uh(:,:,:), vh(:,:,:), G%Domain, halo=2) + call create_group_pass(CS%pass_hp_uv, u_av, v_av, G%Domain, halo=max(2,obc_stencil)) + call create_group_pass(CS%pass_hp_uv, uh(:,:,:), vh(:,:,:), G%Domain, halo=max(2,obc_stencil)) call create_group_pass(CS%pass_uv, u, v, G%Domain, halo=max(2,cont_stencil)) call create_group_pass(CS%pass_h, h, G%Domain, halo=max(2,cont_stencil)) - call create_group_pass(CS%pass_av_uvh, u_av, v_av, G%Domain, halo=2) - call create_group_pass(CS%pass_av_uvh, uh(:,:,:), vh(:,:,:), G%Domain, halo=2) + call create_group_pass(CS%pass_av_uvh, u_av, v_av, G%Domain, halo=max(2,obc_stencil)) + call create_group_pass(CS%pass_av_uvh, uh(:,:,:), vh(:,:,:), G%Domain, halo=max(2,obc_stencil)) call cpu_clock_end(id_clock_pass) !--- end set up for group halo pass From c95bc315b84bf0d1fc9959ccd25d9ea81f04cf49 Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Sun, 8 Oct 2023 20:52:29 -0800 Subject: [PATCH 5/6] Fixed an oopsie with OBC --- src/core/MOM_dynamics_split_RK2.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/MOM_dynamics_split_RK2.F90 b/src/core/MOM_dynamics_split_RK2.F90 index 7e66fa1352..c506d12139 100644 --- a/src/core/MOM_dynamics_split_RK2.F90 +++ b/src/core/MOM_dynamics_split_RK2.F90 @@ -452,7 +452,9 @@ subroutine step_MOM_dyn_split_RK2(u, v, h, tv, visc, Time_local, dt, forces, p_s cont_stencil = continuity_stencil(CS%continuity_CSp) obc_stencil = 2 - if (CS%OBC%oblique_BCs_exist_globally) obc_stencil = 3 + if (associated(CS%OBC)) then + if (CS%OBC%oblique_BCs_exist_globally) obc_stencil = 3 + endif call cpu_clock_begin(id_clock_pass) call create_group_pass(CS%pass_eta, eta, G%Domain, halo=1) call create_group_pass(CS%pass_visc_rem, CS%visc_rem_u, CS%visc_rem_v, G%Domain, & From 33d1a673d6f59d8a75b6c7612cec1fc35660a9b5 Mon Sep 17 00:00:00 2001 From: Kate Hedstrom Date: Mon, 9 Oct 2023 10:12:17 -0800 Subject: [PATCH 6/6] Getting rid of extra exchange (that didn't help) --- src/core/MOM_open_boundary.F90 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core/MOM_open_boundary.F90 b/src/core/MOM_open_boundary.F90 index a0f16b8642..b14f016a0f 100644 --- a/src/core/MOM_open_boundary.F90 +++ b/src/core/MOM_open_boundary.F90 @@ -3294,12 +3294,6 @@ subroutine radiation_open_bdry_conds(OBC, u_new, u_old, v_new, v_old, G, GV, US, call open_boundary_apply_normal_flow(OBC, G, GV, u_new, v_new) call pass_vector(u_new, v_new, G%Domain, clock=id_clock_pass) - if (OBC%oblique_BCs_exist_globally) then - call do_group_pass(OBC%pass_oblique, G%Domain) -! call pass_vector(OBC%rx_oblique_u, OBC%ry_oblique_v, G%Domain, To_All+Scalar_Pair) -! call pass_vector(OBC%ry_oblique_u, OBC%rx_oblique_v, G%Domain, To_All+Scalar_Pair) -! call pass_vector(OBC%cff_normal_u, OBC%cff_normal_v, G%Domain, To_All+Scalar_Pair) - endif end subroutine radiation_open_bdry_conds