Skip to content

Commit

Permalink
Fix read checkpoint for the matrix-free application (#1007)
Browse files Browse the repository at this point in the history 
Description of the problem
The restart option was not working in parallel for the matrix-free application.

Description of the solution
Use locally relevant vectors for the present and previous solutions regardless of the type of vector. In addition, use a locally relevant vector for mesh refinement (in the init_temporary_vector() call).

How Has This Been Tested?
 All existing tests pass.
 A new test tgv_restart_bdf1 is added to the MF application in serial and in parallel to verify restart feature.
  • Loading branch information
@lpsaavedra
lpsaavedra authored Jan 29, 2024
1 parent 591367f commit def88b4
Show file tree
Hide file tree
Showing 13 changed files with 1,157 additions and 10 deletions.
9 changes: 9 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} _build_type)

file(COPY cylinder_structured.msh DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cylinder_kelly_estimator.${_build_type}")

file(COPY tgv_restart_bdf1/restart.pvdhandler DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/restart.simulationcontrol DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/restart.triangulation DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/restart.triangulation_fixed.data DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/restart.triangulation.info DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/enstrophy.checkpoint DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/kinetic_energy.checkpoint DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")
file(COPY tgv_restart_bdf1/L2Error_FD.checkpoint DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tgv_restart_bdf1.${_build_type}")

deal_ii_pickup_tests()

if(CMAKE_BUILD_TYPE STREQUAL "Debug")
Expand Down
461 changes: 461 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1.mpirun=1.output
View file

Large diffs are not rendered by default.

461 changes: 461 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1.mpirun=2.output
View file

Large diffs are not rendered by default.

204 changes: 204 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1.prm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
# Listing of Parameters
#----------------------

set dimension = 2

#---------------------------------------------------
# Simulation Control
#---------------------------------------------------

subsection simulation control
set method = bdf1
set time step = 0.01 # Time step
set number mesh adapt = 0 # If steady, nb mesh adaptation
set time end = 1.00 # End time of simulation
set output frequency = 0 # Frequency of simulation output
set output path = ../
end

#---------------------------------------------------
# FEM
#---------------------------------------------------

subsection FEM
set velocity order = 1
set pressure order = 1
end

#---------------------------------------------------
# Timer
#---------------------------------------------------

subsection timer
set type = none # <none|iteration|end>
end

#---------------------------------------------------
# Initial condition
#---------------------------------------------------

subsection initial conditions
set type = L2projection
subsection uvwp
set Function expression = cos(x)*sin(y); -sin(x)*cos(y); -1./4*(cos(2*x)+cos(2*y));
end
end

#---------------------------------------------------
# Physical Properties
#---------------------------------------------------

subsection physical properties
set number of fluids = 1
subsection fluid 0
set kinematic viscosity = 1.000
end
end

#---------------------------------------------------
# Analytical Solution
#---------------------------------------------------

subsection analytical solution
set verbosity = verbose
set enable = true
subsection uvwp
set Function constants = viscosity=1
set Function expression = exp(-2*viscosity*t)* cos(x) * sin(y) ; -sin(x) * cos(y) * exp(-2*viscosity*t); 0
end
end

#--------------------------------------------------
# Restart
#--------------------------------------------------

subsection restart
# Enable checkpointing
set checkpoint = false
# Prefix for the filename of checkpoints
set filename = restart
# Frequency for checkpointing
set frequency = 1
# Enable restart from checkpoint
set restart = true
end

#---------------------------------------------------
# Post-Processing
#---------------------------------------------------

subsection post-processing
set verbosity = verbose
set calculate enstrophy = true
set calculate kinetic energy = true
end

#---------------------------------------------------
# Mesh
#---------------------------------------------------

subsection mesh
set type = dealii
set grid type = hyper_cube
set grid arguments = 0 : 6.28318530718 : true
set initial refinement = 5 # initial mesh refinement
end

#---------------------------------------------------
# Mesh Adaptation Control
#---------------------------------------------------

subsection mesh adaptation
set type = none
end

#---------------------------------------------------
# Boundary Conditions
#---------------------------------------------------

subsection boundary conditions
set number = 2
subsection bc 0
set type = periodic
set id = 0
set periodic_id = 1
set periodic_direction = 0
end
subsection bc 1
set type = periodic
set id = 2
set periodic_id = 3
set periodic_direction = 1
end
end

#---------------------------------------------------
# Non-Linear Solver Control
#---------------------------------------------------

subsection non-linear solver
subsection fluid dynamics
set verbosity = quiet
set tolerance = 1e-6
set max iterations = 5
end
end

#---------------------------------------------------
# Linear Solver Control
#---------------------------------------------------

subsection linear solver
subsection fluid dynamics
set verbosity = quiet
set method = gmres
set max iters = 5000
set relative residual = 1e-4
set minimum residual = 1e-9
set preconditioner = ilu
set ilu preconditioner fill = 0
set ilu preconditioner absolute tolerance = 1e-10
set ilu preconditioner relative tolerance = 1.00
end
end

#---------------------------------------------------
# Linear Solver Control
#---------------------------------------------------

subsection linear solver
subsection fluid dynamics
set method = gmres
set max iters = 5000
set relative residual = 1e-4
set minimum residual = 1e-7
set preconditioner = gcmg
set verbosity = quiet

#MG parameters
set mg verbosity = quiet
set mg min level = -1
set mg level min cells = 16

#smoother
set mg smoother iterations = 10
set mg smoother eig estimation = true

# Eigenvalue estimation parameters
set eig estimation degree = 3
set eig estimation smoothing range = 5
set eig estimation cg n iterations = 20
set eig estimation verbosity = quiet

#coarse-grid solver
set mg coarse grid max iterations = 2000
set mg coarse grid tolerance = 1e-7
set mg coarse grid reduce = 1e-4
set mg coarse grid max krylov vectors = 30
set mg coarse grid preconditioner = ilu

set ilu preconditioner fill = 1
set ilu preconditioner absolute tolerance = 1e-10
set ilu preconditioner relative tolerance = 1.00
end
end
1 change: 1 addition & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/L2Error_FD.checkpoint
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0 0 0 0 2 0 4 time 14 error_velocity 0 0 2 0 0 0 14 error_velocity 0 0 0 0 50 0 0 0 100 9.70260438756851947e-04 100 1.37351724439845491e-03 100 1.84010607302818440e-03 100 2.31749994121434625e-03 100 2.78876322761135520e-03 100 3.24754270524175775e-03 100 3.69120352527292075e-03 100 4.11864350666947716e-03 100 4.52947302130056548e-03 100 4.92366418756171736e-03 100 5.30138467716634483e-03 100 5.66291238323648246e-03 100 6.00858869287675037e-03 100 6.33879153008128950e-03 100 6.65391912906595270e-03 100 6.95437992264262127e-03 100 7.24058606285793438e-03 100 7.51294917727212980e-03 100 7.77187754425011975e-03 100 8.01777419334964020e-03 100 8.25103562307126157e-03 100 8.47205093909883522e-03 100 8.68120128407522638e-03 100 8.87885947266102427e-03 100 9.06538977306088449e-03 100 9.24114779421839234e-03 100 9.40648044994206629e-03 100 9.56172597941684979e-03 100 9.70721400924101641e-03 100 9.84326564608971506e-03 100 9.97019359194816109e-03 100 1.00883022758919815e-02 100 1.01978879978666098e-02 100 1.02992390810181089e-02 100 1.03926360299361770e-02 100 1.04783516927750304e-02 100 1.05566514256825311e-02 100 1.06277932583167135e-02 100 1.06920280595025063e-02 100 1.07495997022842796e-02 100 1.08007452277957966e-02 100 1.08456950075020674e-02 100 1.08846729034549842e-02 100 1.09178964262987305e-02 100 1.09455768908119956e-02 100 1.09679195688370847e-02 100 1.09851238394703850e-02 100 1.09973833364404670e-02 100 1.10048860926087438e-02 100 1.10078146815611386e-02 14 error_velocity 1 c 4 0 0 6 4 time 50 0 100 1.00000000000000002e-02 100 2.00000000000000004e-02 100 2.99999999999999989e-02 100 4.00000000000000008e-02 100 5.00000000000000028e-02 100 6.00000000000000047e-02 100 7.00000000000000067e-02 100 8.00000000000000017e-02 100 8.99999999999999967e-02 100 9.99999999999999917e-02 100 1.09999999999999987e-01 100 1.19999999999999982e-01 100 1.29999999999999977e-01 100 1.39999999999999986e-01 100 1.49999999999999994e-01 100 1.60000000000000003e-01 100 1.70000000000000012e-01 100 1.80000000000000021e-01 100 1.90000000000000030e-01 100 2.00000000000000039e-01 100 2.10000000000000048e-01 100 2.20000000000000057e-01 100 2.30000000000000066e-01 100 2.40000000000000074e-01 100 2.50000000000000056e-01 100 2.60000000000000064e-01 100 2.70000000000000073e-01 100 2.80000000000000082e-01 100 2.90000000000000091e-01 100 3.00000000000000100e-01 100 3.10000000000000109e-01 100 3.20000000000000118e-01 100 3.30000000000000127e-01 100 3.40000000000000135e-01 100 3.50000000000000144e-01 100 3.60000000000000153e-01 100 3.70000000000000162e-01 100 3.80000000000000171e-01 100 3.90000000000000180e-01 100 4.00000000000000189e-01 100 4.10000000000000198e-01 100 4.20000000000000207e-01 100 4.30000000000000215e-01 100 4.40000000000000224e-01 100 4.50000000000000233e-01 100 4.60000000000000242e-01 100 4.70000000000000251e-01 100 4.80000000000000260e-01 100 4.90000000000000269e-01 100 5.00000000000000000e-01 4 time 1 c 4 0 0 6 0 0 0 0 0 0 0 0 0 0 0
1 change: 1 addition & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/enstrophy.checkpoint
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0 0 0 0 2 0 4 time 9 enstrophy 0 0 2 0 0 0 9 enstrophy 0 0 0 0 51 0 0 0 100 5.00800056059643395e-01 100 4.81292453184982871e-01 100 4.62544734156999260e-01 100 4.44527299704690182e-01 100 4.27211702324144738e-01 100 4.10570602690655428e-01 100 3.94577726493181846e-01 100 3.79207822943402106e-01 100 3.64436624901343820e-01 100 3.50240810554676485e-01 100 3.36597966591067610e-01 100 3.23486552805436200e-01 100 3.10885868086191564e-01 100 2.98776017726692045e-01 100 2.87137882010295609e-01 100 2.75953086019365912e-01 100 2.65203970620546070e-01 100 2.54873564580438028e-01 100 2.44945557767675820e-01 100 2.35404275399022533e-01 100 2.26234653288830939e-01 100 2.17422214062759800e-01 100 2.08953044298186313e-01 100 2.00813772555184233e-01 100 1.92991548263393459e-01 100 1.85474021431407649e-01 100 1.78249323146645405e-01 100 1.71306046834909798e-01 100 1.64633230250013057e-01 100 1.58220338165055374e-01 100 1.52057245737990759e-01 100 1.46134222525220264e-01 100 1.40441917117969300e-01 100 1.34971342377177811e-01 100 1.29713861243581335e-01 100 1.24661173100587941e-01 100 1.19805300668398645e-01 100 1.15138577408682025e-01 100 1.10653635419910273e-01 100 1.06343393804241890e-01 100 1.02201047487581465e-01 100 9.82200564751545685e-02 100 9.43941355256398690e-02 100 9.07172442275475249e-02 100 8.71835774621714382e-02 100 8.37875562380576699e-02 100 8.05238188825218026e-02 100 7.73872125762883750e-02 100 7.43727852179072585e-02 100 7.14757776050800875e-02 100 6.86916159205671339e-02 9 enstrophy 1 c 12 0 0 14 4 time 51 0 100 0.00000000000000000e+00 100 1.00000000000000002e-02 100 2.00000000000000004e-02 100 2.99999999999999989e-02 100 4.00000000000000008e-02 100 5.00000000000000028e-02 100 6.00000000000000047e-02 100 7.00000000000000067e-02 100 8.00000000000000017e-02 100 8.99999999999999967e-02 100 9.99999999999999917e-02 100 1.09999999999999987e-01 100 1.19999999999999982e-01 100 1.29999999999999977e-01 100 1.39999999999999986e-01 100 1.49999999999999994e-01 100 1.60000000000000003e-01 100 1.70000000000000012e-01 100 1.80000000000000021e-01 100 1.90000000000000030e-01 100 2.00000000000000039e-01 100 2.10000000000000048e-01 100 2.20000000000000057e-01 100 2.30000000000000066e-01 100 2.40000000000000074e-01 100 2.50000000000000056e-01 100 2.60000000000000064e-01 100 2.70000000000000073e-01 100 2.80000000000000082e-01 100 2.90000000000000091e-01 100 3.00000000000000100e-01 100 3.10000000000000109e-01 100 3.20000000000000118e-01 100 3.30000000000000127e-01 100 3.40000000000000135e-01 100 3.50000000000000144e-01 100 3.60000000000000153e-01 100 3.70000000000000162e-01 100 3.80000000000000171e-01 100 3.90000000000000180e-01 100 4.00000000000000189e-01 100 4.10000000000000198e-01 100 4.20000000000000207e-01 100 4.30000000000000215e-01 100 4.40000000000000224e-01 100 4.50000000000000233e-01 100 4.60000000000000242e-01 100 4.70000000000000251e-01 100 4.80000000000000260e-01 100 4.90000000000000269e-01 100 5.00000000000000000e-01 4 time 1 c 12 0 0 14 0 0 0 0 0 0 0 0 0 0 0
1 change: 1 addition & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/kinetic_energy.checkpoint
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0 0 0 0 2 0 4 time 14 kinetic-energy 0 0 2 0 0 0 14 kinetic-energy 0 0 0 0 51 0 0 0 100 2.49999992602278648e-01 100 2.40261774191911220e-01 100 2.30902890185604948e-01 100 2.21908565118993956e-01 100 2.13264597968605696e-01 100 2.04957340910372487e-01 100 1.96973677776323297e-01 100 1.89301003342272928e-01 100 1.81927203422487666e-01 100 1.74840635739944933e-01 100 1.68030111541929122e-01 100 1.61484877931928866e-01 100 1.55194600889924122e-01 100 1.49149348954229294e-01 100 1.43339577539106894e-01 100 1.37756113863395757e-01 100 1.32390142466314364e-01 100 1.27233191287579483e-01 100 1.22277118289848358e-01 100 1.17514098602333875e-01 100 1.12936612165310174e-01 100 1.08537431855967204e-01 100 1.04309612076881122e-01 100 1.00246477789047023e-01 100 9.63416139721800652e-02 100 9.25888554956175447e-02 100 8.89822773838300912e-02 100 8.55161854611709293e-02 100 8.21851073610761068e-02 100 7.89837838855261404e-02 100 7.59071607011192057e-02 100 7.29503803586381838e-02 100 7.01087746235089238e-02 100 6.73778571050381719e-02 100 6.47533161727949780e-02 100 6.22310081489358957e-02 100 5.98069507657364163e-02 100 5.74773168779894733e-02 100 5.52384284203424891e-02 100 5.30867506000306175e-02 100 5.10188863158367312e-02 100 4.90315707944623119e-02 100 4.71216664358406687e-02 100 4.52861578592531691e-02 100 4.35221471424239587e-02 100 4.18268492460759922e-02 100 4.01975876167232557e-02 100 3.86317899607558635e-02 100 3.71269841831440434e-02 100 3.56807944843488614e-02 100 3.42909376092767615e-02 14 kinetic-energy 1 c 12 0 0 14 4 time 51 0 100 0.00000000000000000e+00 100 1.00000000000000002e-02 100 2.00000000000000004e-02 100 2.99999999999999989e-02 100 4.00000000000000008e-02 100 5.00000000000000028e-02 100 6.00000000000000047e-02 100 7.00000000000000067e-02 100 8.00000000000000017e-02 100 8.99999999999999967e-02 100 9.99999999999999917e-02 100 1.09999999999999987e-01 100 1.19999999999999982e-01 100 1.29999999999999977e-01 100 1.39999999999999986e-01 100 1.49999999999999994e-01 100 1.60000000000000003e-01 100 1.70000000000000012e-01 100 1.80000000000000021e-01 100 1.90000000000000030e-01 100 2.00000000000000039e-01 100 2.10000000000000048e-01 100 2.20000000000000057e-01 100 2.30000000000000066e-01 100 2.40000000000000074e-01 100 2.50000000000000056e-01 100 2.60000000000000064e-01 100 2.70000000000000073e-01 100 2.80000000000000082e-01 100 2.90000000000000091e-01 100 3.00000000000000100e-01 100 3.10000000000000109e-01 100 3.20000000000000118e-01 100 3.30000000000000127e-01 100 3.40000000000000135e-01 100 3.50000000000000144e-01 100 3.60000000000000153e-01 100 3.70000000000000162e-01 100 3.80000000000000171e-01 100 3.90000000000000180e-01 100 4.00000000000000189e-01 100 4.10000000000000198e-01 100 4.20000000000000207e-01 100 4.30000000000000215e-01 100 4.40000000000000224e-01 100 4.50000000000000233e-01 100 4.60000000000000242e-01 100 4.70000000000000251e-01 100 4.80000000000000260e-01 100 4.90000000000000269e-01 100 5.00000000000000000e-01 4 time 1 c 12 0 0 14 0 0 0 0 0 0 0 0 0 0 0
2 changes: 2 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/restart.pvdhandler
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
0
Time File
8 changes: 8 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/restart.simulationcontrol
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Simulation control
dt_0 0.01
dt_1 0.01
dt_2 0.01
dt_3 0.01
CFL 0.0166875
Time 0.5
Iter 50
Binary file added applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/restart.triangulation
View file
Binary file not shown.
2 changes: 2 additions & 0 deletions applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/restart.triangulation.info
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
version nproc n_attached_fixed_size_objs n_attached_variable_size_objs n_coarse_cells
5 1 1 0 1
Binary file added applications_tests/lethe-fluid-matrix-free/tgv_restart_bdf1/restart.triangulation_fixed.data
View file
Binary file not shown.
17 changes: 7 additions & 10 deletions source/solvers/navier_stokes_base.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1576,15 +1576,19 @@ NavierStokesBase<dim, VectorType, DofsType>::read_checkpoint()
setup_dofs();
std::vector<VectorType *> x_system(1 + previous_solutions.size());

VectorType distributed_system(locally_owned_dofs, this->mpi_communicator);
VectorType distributed_system(locally_owned_dofs,
this->locally_relevant_dofs,
this->mpi_communicator);
x_system[0] = &(distributed_system);

std::vector<VectorType> distributed_previous_solutions;
distributed_previous_solutions.reserve(previous_solutions.size());
for (unsigned int i = 0; i < previous_solutions.size(); ++i)
{
distributed_previous_solutions.emplace_back(
VectorType(locally_owned_dofs, this->mpi_communicator));
VectorType(locally_owned_dofs,
this->locally_relevant_dofs,
this->mpi_communicator));
x_system[i + 1] = &distributed_previous_solutions[i];
}
parallel::distributed::SolutionTransfer<dim, VectorType> system_trans_vectors(
Expand Down Expand Up @@ -2336,15 +2340,8 @@ NavierStokesBase<dim, VectorType, DofsType>::init_temporary_vector()
{
VectorType tmp;

if constexpr (std::is_same_v<VectorType, GlobalVectorType> ||
std::is_same_v<VectorType, GlobalBlockVectorType>)
tmp.reinit(locally_owned_dofs, this->mpi_communicator);
tmp.reinit(locally_owned_dofs, locally_relevant_dofs, this->mpi_communicator);

else if constexpr (std::is_same_v<VectorType,
LinearAlgebra::distributed::Vector<double>>)
tmp.reinit(locally_owned_dofs,
locally_relevant_dofs,
this->mpi_communicator);
return tmp;
}

Expand Down

0 comments on commit def88b4

Please sign in to comment.