Skip to content

Memory checking utility for CCPP builds (of NEMSfv3gfs) #127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
climbfuji merged 3 commits into from
Aug 22, 2018
Merged

Memory checking utility for CCPP builds (of NEMSfv3gfs) #127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
23f7c77
Add memcheck scheme to CCPP prebuild config for FV3, and suite defini…
climbfuji Aug 21, 2018
c4811d4
Merge branch 'master' of https://github.com/NCAR/ccpp-framework into …
climbfuji Aug 22, 2018
5ddb31c
src/ccpp_memory.c: gather and print information about OpenMP threads
climbfuji Aug 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions examples/suite_FV3_test_memcheck.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8"?>

<suite name="FV3_test" lib="ccppphys" ver="1">
<!-- <init></init> -->
<group name="time_vary">
<subcycle loop="1">
<scheme>GFS_time_vary_pre</scheme>
<scheme>GFS_rrtmg_setup</scheme>
<scheme>GFS_rad_time_vary</scheme>
<scheme>GFS_phys_time_vary</scheme>
<scheme>stochastic_physics</scheme>
<scheme>stochastic_physics_sfc</scheme>
</subcycle>
</group>
<group name="radiation">
<subcycle loop="1">
<scheme>GFS_suite_interstitial_rad_reset</scheme>
<scheme>GFS_rrtmg_pre</scheme>
<scheme>rrtmg_sw_pre</scheme>
<scheme>rrtmg_sw</scheme>
<scheme>rrtmg_sw_post</scheme>
<scheme>rrtmg_lw_pre</scheme>
<scheme>rrtmg_lw</scheme>
<scheme>rrtmg_lw_post</scheme>
<scheme>GFS_rrtmg_post</scheme>
<!-- <scheme>memcheck</scheme> -->
<!-- <scheme>GFS_diagtoscreen</scheme> -->
<!-- <scheme>GFS_interstitialtoscreen</scheme> -->
</subcycle>
</group>
<group name="physics">
<subcycle loop="1">
<scheme>GFS_suite_interstitial_phys_reset</scheme>
<scheme>GFS_suite_stateout_reset</scheme>
<scheme>get_prs_fv3</scheme>
<scheme>GFS_suite_interstitial_1</scheme>
<scheme>dcyc2t3</scheme>
<scheme>GFS_surface_generic_pre</scheme>
</subcycle>
<!-- Surface iteration loop -->
<subcycle loop="2">
<scheme>sfc_ex_coef</scheme>
<scheme>GFS_surface_loop_control_part1</scheme>
<scheme>sfc_nst_pre</scheme>
<scheme>sfc_nst</scheme>
<scheme>sfc_nst_post</scheme>
<scheme>lsm_noah</scheme>
<scheme>sfc_sice</scheme>
<scheme>GFS_surface_loop_control_part2</scheme>
</subcycle>
<!-- End of surface iteration loop -->
<subcycle loop="1">
<scheme>dcyc2t3_post</scheme>
<scheme>sfc_diag</scheme>
<scheme>GFS_surface_generic_post</scheme>
<scheme>GFS_PBL_generic_pre</scheme>
<scheme>hedmf</scheme>
<scheme>GFS_PBL_generic_post</scheme>
<scheme>gwdps_pre</scheme>
<scheme>gwdps</scheme>
<scheme>gwdps_post</scheme>
<scheme>rayleigh_damp</scheme>
<scheme>ozphys</scheme>
<scheme>GFS_DCNV_generic_pre</scheme>
<scheme>get_phi_fv3</scheme>
<scheme>GFS_suite_interstitial_3</scheme>
<scheme>samfdeepcnv</scheme>
<scheme>GFS_DCNV_generic_post</scheme>
<scheme>memcheck</scheme>
<scheme>gwdc_pre</scheme>
<scheme>gwdc</scheme>
<scheme>gwdc_post</scheme>
<scheme>GFS_SCNV_generic_pre</scheme>
<scheme>samfshalcnv</scheme>
<scheme>samfshalcnv_post</scheme>
<scheme>GFS_SCNV_generic_post</scheme>
<scheme>GFS_suite_interstitial_4</scheme>
<scheme>cnvc90</scheme>
<scheme>zhaocarr_gscond</scheme>
<scheme>zhaocarr_precpd</scheme>
<scheme>GFS_MP_generic_post</scheme>
<scheme>sfc_sice_post</scheme>
</subcycle>
</group>
<group name="stochastics">
<subcycle loop="1">
<scheme>GFS_stochastics</scheme>
</subcycle>
</group>
<!-- <finalize></finalize> -->
</suite>
5 changes: 5 additions & 0 deletions scripts/ccpp_prebuild_config_FV3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@
'ccpp/physics/physics/sfc_sice.f' : [ 'slow_physics' ],
# stochastic physics
'ccpp/physics/stochastic_physics/stochastic_physics.F90' : [ 'slow_physics' ],
# memcheck utility
'ccpp/physics/physics/memcheck.F90' : [ 'slow_physics' ],
}

# Auto-generated makefile/cmakefile snippets that contain all schemes
Expand Down Expand Up @@ -215,6 +217,9 @@
'mean_effective_radius_for_snow_flake',
],
},
'memcheck' : {
'memcheck_run' : [ 'mpi_root' ],
},
#'subroutine_name_1' : 'all',
#'subroutine_name_2' : 'none',
#'subroutine_name_2' : [ 'var1', 'var3'],
Expand Down
45 changes: 36 additions & 9 deletions src/ccpp_memory.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
#ifdef MPI
#include <mpi.h>
#endif
#ifdef OPENMP
#include <omp.h>
#endif

#include "ccpp_memory.h"

Expand Down Expand Up @@ -153,6 +156,17 @@ int ccpp_memory_usage_c(const int mpicomm, char* str, int lstr){
mpisize = 1;
#endif

// Get OpenMP thread number and total number of threads
int ompthread;
int ompthreads;
#ifdef OPENMP
ompthread = omp_get_thread_num();
ompthreads = omp_get_num_threads();
#else
ompthread = 0;
ompthreads = 1;
#endif

// Retrieve hostname
char hostname[128];
gethostname(hostname, sizeof(hostname));
Expand Down Expand Up @@ -211,7 +225,7 @@ int ccpp_memory_usage_c(const int mpicomm, char* str, int lstr){
#ifndef MPI
snprintf(strtmp, sizeof(strtmp),
"--------------------------------------------------------------------------------\n\
Memory usage - MPI rank %d/%d\n\
Memory usage - MPI rank %d/%d - OMP thread %d/%d\n\
\n\
Statistics for node %s:\n\
Total virtual memory: %lld MB\n\
Expand All @@ -224,12 +238,13 @@ Virtual memory this process: %d MB\n\
Physical memory this process: %d MB\n\
Max. phys. memory this process: %d MB\n\
--------------------------------------------------------------------------------\n\
", mpirank, mpisize, hostname, totalVirtMem, virtMemUsed, totalPhysMem, physMemUsed,
", mpirank, mpisize, ompthread, ompthreads,
hostname, totalVirtMem, virtMemUsed, totalPhysMem, physMemUsed,
pid, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax);
#else
snprintf(strtmp, sizeof(strtmp),
"--------------------------------------------------------------------------------\n\
Memory usage - MPI rank %d/%d\n\
Memory usage - MPI rank %d/%d - OMP thread %d/%d\n\
\n\
Statistics for node %s:\n\
Total virtual memory: %lld MB\n\
Expand All @@ -247,7 +262,8 @@ Virtual memory average: %d MB\n\
Physical memory average: %d MB\n\
Max. phys. memory average: %d MB\n\
--------------------------------------------------------------------------------\n\
", mpirank, mpisize, hostname, totalVirtMem, virtMemUsed, totalPhysMem, physMemUsed,
", mpirank, mpisize, ompthread, ompthreads,
hostname, totalVirtMem, virtMemUsed, totalPhysMem, physMemUsed,
pid, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax, virtMemPerProcessAvg,
physMemPerProcessAvg, physMemPerProcessMaxAvg);
#endif
Expand Down Expand Up @@ -294,7 +310,16 @@ int ccpp_memory_usage_c(const int mpicomm, char* str, int lstr){
mpisize = 1;
#endif

// Retrieve memory statistics
// Get OpenMP thread number and total number of threads
int ompthread;
int ompthreads;
#ifdef OPENMP
ompthread = omp_get_thread_num();
ompthreads = omp_get_num_threads();
#else
ompthread = 0;
ompthreads = 1;
#endif

// Available physical memory - in Byte
unsigned long long totalPhysMem;
Expand Down Expand Up @@ -338,19 +363,20 @@ int ccpp_memory_usage_c(const int mpicomm, char* str, int lstr){
#ifndef MPI
snprintf(strtmp, sizeof(strtmp),
"--------------------------------------------------------------------------------\n\
Memory usage - MPI rank %d/%d\n\
Memory usage - MPI rank %d/%d - OMP thread %d/%d\n\
\n\
Size of physical memory: %llu MB\n\
\n\
Virtual memory this process: %d MB\n\
Physical memory this process: %d MB\n\
Max. phys. memory this process: %d MB\n\
--------------------------------------------------------------------------------\n\
", mpirank, mpisize, totalPhysMem, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax);
", mpirank, mpisize, ompthread, ompthreads,
totalPhysMem, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax);
#else
snprintf(strtmp, sizeof(strtmp),
"--------------------------------------------------------------------------------\n\
Memory usage - MPI rank %d/%d\n\
Memory usage - MPI rank %d/%d - OMP thread %d/%d\n\
\n\
Size of physical memory: %llu MB\n\
\n\
Expand All @@ -363,7 +389,8 @@ Virtual memory average: %d MB\n\
Physical memory average: %d MB\n\
Max. phys. memory average: %d MB\n\
--------------------------------------------------------------------------------\n\
", mpirank, mpisize, totalPhysMem, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax,
", mpirank, mpisize, ompthread, ompthreads,
totalPhysMem, virtMemPerProcess, physMemPerProcess, physMemPerProcessMax,
virtMemPerProcessAvg, physMemPerProcessAvg, physMemPerProcessMaxAvg);
#endif

Expand Down