diff --git a/pyphare/pyphare/pharein/simulation.py b/pyphare/pyphare/pharein/simulation.py index 697e6a0ee..684c07084 100644 --- a/pyphare/pyphare/pharein/simulation.py +++ b/pyphare/pyphare/pharein/simulation.py @@ -401,12 +401,16 @@ def get_max_ghosts(): grid.nbrGhosts(kwargs["interp_order"], x) for x in ["primal", "dual"] ) + interp = kwargs["interp_order"] max_ghosts = get_max_ghosts() small_invalid_patch_size = phare_utilities.np_array_ify(max_ghosts, ndim) largest_patch_size = kwargs.get("largest_patch_size", None) - # to prevent primal ghost overlaps of non adjacent patches, we need smallest_patch_size+=1 - smallest_patch_size = phare_utilities.np_array_ify(max_ghosts, ndim) + 1 + # to prevent primal ghost box overlaps of non adjacent patches, we need smallest_patch_size * 2 + 1 + smallest_patch_size = phare_utilities.np_array_ify(max_ghosts, ndim) * 2 + 1 + # TORM next lines after https://github.com/llnl/SAMRAI/issues/311 + min_per_interp = [6, 9, 9] # SAMRAI BORDER BUG + smallest_patch_size = phare_utilities.np_array_ify(min_per_interp[interp - 1], ndim) if "smallest_patch_size" in kwargs and kwargs["smallest_patch_size"] is not None: smallest_patch_size = phare_utilities.np_array_ify( kwargs["smallest_patch_size"], ndim diff --git a/pyphare/pyphare/simulator/simulator.py b/pyphare/pyphare/simulator/simulator.py index ada192774..d7fdd4e11 100644 --- a/pyphare/pyphare/simulator/simulator.py +++ b/pyphare/pyphare/simulator/simulator.py @@ -75,7 +75,7 @@ class Simulator: These arguments have good default, change them at your own risk. - * **print_one_line**: (``bool``), default True, will print simulator info per advance on one line (erasing the previous) + * **print_one_line**: (``bool``), default False, will print simulator info per advance on one line (erasing the previous) * **auto_dump**: (``bool``), if True (default), will dump diagnostics automatically at requested timestamps * **post_advance**: (``Function``), default None. A python function to execute after each advance() * **log_to_file**: if True (default), will log prints made from C++ code per MPI rank to the .log directory @@ -91,7 +91,7 @@ def __init__(self, simulation, auto_dump=True, **kwargs): self.post_advance = kwargs.get("post_advance", None) self.initialized = False self.print_eol = "\n" - if kwargs.get("print_one_line", True): + if kwargs.get("print_one_line", False): self.print_eol = "\r" self.print_eol = kwargs.get("print_eol", self.print_eol) self.log_to_file = kwargs.get("log_to_file", True) diff --git a/src/amr/data/field/field_data.hpp b/src/amr/data/field/field_data.hpp index aa0938071..8699cc80c 100644 --- a/src/amr/data/field/field_data.hpp +++ b/src/amr/data/field/field_data.hpp @@ -27,8 +27,12 @@ namespace amr typename PhysicalQuantity = decltype(std::declval().physicalQuantity())> class FieldData : public SAMRAI::hier::PatchData { - using Super = SAMRAI::hier::PatchData; + using Super = SAMRAI::hier::PatchData; + + public: using value_type = Grid_t::value_type; + + private: using SetEqualOp = core::Equals; public: @@ -98,7 +102,7 @@ namespace amr * The data will be copied from the interior and ghost of the source to the interior and * ghost of the destination, where there is an overlap in the underlying index space */ - void copy(const SAMRAI::hier::PatchData& source) final + void copy(SAMRAI::hier::PatchData const& source) final { PHARE_LOG_SCOPE(3, "FieldData::copy"); @@ -154,8 +158,8 @@ namespace amr * give the necessary transformation to apply to the source, to perform the copy (ie : * translation for periodics condition) */ - void copy(const SAMRAI::hier::PatchData& source, - const SAMRAI::hier::BoxOverlap& overlap) final + void copy(SAMRAI::hier::PatchData const& source, + SAMRAI::hier::BoxOverlap const& overlap) final { PHARE_LOG_SCOPE(3, "FieldData::copy"); @@ -172,7 +176,7 @@ namespace amr /*** \brief This form should not be called since we cannot derive from FieldData */ void copy2([[maybe_unused]] SAMRAI::hier::PatchData& destination, - [[maybe_unused]] const SAMRAI::hier::BoxOverlap& overlap) const final + [[maybe_unused]] SAMRAI::hier::BoxOverlap const& overlap) const final { throw std::runtime_error("Error cannot cast the PatchData to FieldData"); } @@ -193,7 +197,7 @@ namespace amr /*** \brief Compute the maximum amount of memory needed to hold FieldData information on * the specified overlap */ - std::size_t getDataStreamSize(const SAMRAI::hier::BoxOverlap& overlap) const final + std::size_t getDataStreamSize(SAMRAI::hier::BoxOverlap const& overlap) const final { return getDataStreamSize_(overlap); } @@ -205,7 +209,7 @@ namespace amr * overlap, and put it on the stream. */ void packStream(SAMRAI::tbox::MessageStream& stream, - const SAMRAI::hier::BoxOverlap& overlap) const final + SAMRAI::hier::BoxOverlap const& overlap) const final { PHARE_LOG_SCOPE(3, "packStream"); @@ -243,14 +247,14 @@ namespace amr * by the overlap, and fill the data where is needed. */ void unpackStream(SAMRAI::tbox::MessageStream& stream, - const SAMRAI::hier::BoxOverlap& overlap) final + SAMRAI::hier::BoxOverlap const& overlap) final { unpackStream(stream, overlap, field); } template void unpackStream(SAMRAI::tbox::MessageStream& stream, - const SAMRAI::hier::BoxOverlap& overlap, Grid_t& dst_grid) + SAMRAI::hier::BoxOverlap const& overlap, Grid_t& dst_grid) { PHARE_LOG_SCOPE(3, "unpackStream"); @@ -308,9 +312,14 @@ namespace amr } - void sum(SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap); - void unpackStreamAndSum(SAMRAI::tbox::MessageStream& stream, - SAMRAI::hier::BoxOverlap const& overlap); + template + void operate(SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap); + + + template + void unpackStreamAnd(SAMRAI::tbox::MessageStream& stream, + SAMRAI::hier::BoxOverlap const& overlap); + GridLayoutT gridLayout; Grid_t field; @@ -367,9 +376,6 @@ namespace amr { SAMRAI::hier::BoxContainer const& boxList = overlap.getDestinationBoxContainer(); - SAMRAI::hier::IntVector const zeroOffset{ - SAMRAI::hier::IntVector::getZero(SAMRAI::tbox::Dimension{dimension})}; - if (transformation.getBeginBlock() == transformation.getEndBlock()) { for (auto const& box : boxList) @@ -434,28 +440,24 @@ namespace PHARE::amr template -void FieldData::unpackStreamAndSum( +template +void FieldData::unpackStreamAnd( SAMRAI::tbox::MessageStream& stream, SAMRAI::hier::BoxOverlap const& overlap) { - using PlusEqualOp = core::PlusEquals; - - unpackStream(stream, overlap, field); + unpackStream(stream, overlap, field); } - - template -void FieldData::sum(SAMRAI::hier::PatchData const& src, - SAMRAI::hier::BoxOverlap const& overlap) +template +void FieldData::operate( + SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap) { - using PlusEqualOp = core::PlusEquals; - TBOX_ASSERT_OBJDIM_EQUALITY2(*this, src); auto& fieldOverlap = dynamic_cast(overlap); auto& fieldSource = dynamic_cast(src); - copy_(fieldSource, fieldOverlap, field); + copy_(fieldSource, fieldOverlap, field); } diff --git a/src/amr/data/tensorfield/tensor_field_data.hpp b/src/amr/data/tensorfield/tensor_field_data.hpp index 0389be30b..65ea55b9f 100644 --- a/src/amr/data/tensorfield/tensor_field_data.hpp +++ b/src/amr/data/tensorfield/tensor_field_data.hpp @@ -46,7 +46,10 @@ class TensorFieldData : public SAMRAI::hier::PatchData [&](auto i) { return Grid_t{compNames[i], qts[i], layout.allocSize(qts[i])}; }); } +public: using value_type = Grid_t::value_type; + +private: using SetEqualOp = core::Equals; public: @@ -336,10 +339,12 @@ class TensorFieldData : public SAMRAI::hier::PatchData return patchData->grids; } - void sum(SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap); - void unpackStreamAndSum(SAMRAI::tbox::MessageStream& stream, - SAMRAI::hier::BoxOverlap const& overlap); + template + void operate(SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap); + template + void unpackStreamAnd(SAMRAI::tbox::MessageStream& stream, + SAMRAI::hier::BoxOverlap const& overlap); GridLayoutT gridLayout; @@ -481,30 +486,26 @@ class TensorFieldData : public SAMRAI::hier::PatchData - template -void TensorFieldData::unpackStreamAndSum( +template +void TensorFieldData::unpackStreamAnd( SAMRAI::tbox::MessageStream& stream, SAMRAI::hier::BoxOverlap const& overlap) { - using PlusEqualOp = core::PlusEquals; - - unpackStream(stream, overlap, grids); + unpackStream(stream, overlap, grids); } - template -void TensorFieldData::sum( +template +void TensorFieldData::operate( SAMRAI::hier::PatchData const& src, SAMRAI::hier::BoxOverlap const& overlap) { - using PlusEqualOp = core::PlusEquals; - TBOX_ASSERT_OBJDIM_EQUALITY2(*this, src); auto& fieldOverlap = dynamic_cast(overlap); auto& fieldSource = dynamic_cast(src); - copy_(fieldSource, fieldOverlap, *this); + copy_(fieldSource, fieldOverlap, *this); } diff --git a/src/amr/level_initializer/hybrid_level_initializer.hpp b/src/amr/level_initializer/hybrid_level_initializer.hpp index a6429d3e1..0249e3db6 100644 --- a/src/amr/level_initializer/hybrid_level_initializer.hpp +++ b/src/amr/level_initializer/hybrid_level_initializer.hpp @@ -128,6 +128,7 @@ namespace solver ions.computeChargeDensity(); ions.computeBulkVelocity(); } + hybMessenger.fillIonBorders(ions, level, initDataTime); // on level i>0, this relies on 'prepareStep' having been called on when // level i-1 was initialized (at the end of this function) @@ -145,38 +146,36 @@ namespace solver // this only needs to be done for the root level // since otherwise initLevel has done it already // TODO NICO comment! E is regridded, we only needed J for E + if (!isRegriddingL0) if (isRootLevel(levelNumber)) { auto& B = hybridModel.state.electromag.B; auto& J = hybridModel.state.J; - for (auto& patch : level) + for (auto& patch : rm.enumerate(level, B, J)) { - auto _ = hybridModel.resourcesManager->setOnPatch(*patch, B, J); auto layout = PHARE::amr::layoutFromPatch(*patch); auto __ = core::SetLayout(&layout, ampere_); ampere_(B, J); - hybridModel.resourcesManager->setTime(J, *patch, 0.); + rm.setTime(J, *patch, 0.); } hybMessenger.fillCurrentGhosts(J, level, 0.); auto& electrons = hybridModel.state.electrons; auto& E = hybridModel.state.electromag.E; - for (auto& patch : level) + for (auto& patch : rm.enumerate(level, B, E, J, electrons)) { auto layout = PHARE::amr::layoutFromPatch(*patch); - auto _ - = hybridModel.resourcesManager->setOnPatch(*patch, B, E, J, electrons); electrons.update(layout); auto& Ve = electrons.velocity(); auto& Ne = electrons.density(); auto& Pe = electrons.pressure(); auto __ = core::SetLayout(&layout, ohm_); ohm_(Ne, Ve, Pe, B, J, E); - hybridModel.resourcesManager->setTime(E, *patch, 0.); + rm.setTime(E, *patch, 0.); } hybMessenger.fillElectricGhosts(E, level, 0.); diff --git a/src/amr/messengers/field_sum_transaction.hpp b/src/amr/messengers/field_operate_transaction.hpp similarity index 57% rename from src/amr/messengers/field_sum_transaction.hpp rename to src/amr/messengers/field_operate_transaction.hpp index 022b9bd14..da128b9fa 100644 --- a/src/amr/messengers/field_sum_transaction.hpp +++ b/src/amr/messengers/field_operate_transaction.hpp @@ -1,5 +1,5 @@ -#ifndef PHARE_AMR_MESSENGERS_FIELD_SUM_TRANSACTION_HPP -#define PHARE_AMR_MESSENGERS_FIELD_SUM_TRANSACTION_HPP +#ifndef PHARE_AMR_MESSENGERS_FIELD_OP_TRANSACTION_HPP +#define PHARE_AMR_MESSENGERS_FIELD_OP_TRANSACTION_HPP #include "core/logger.hpp" @@ -14,31 +14,30 @@ namespace PHARE::amr { -/** * @brief FieldBorderSumTransaction is used to += pop density and flux on ghost box overlaps +/** * @brief FieldBorderOpTransaction is provided to perform operations on border data shared + * across adjacent patches with overlapping ghost boxes * - * A FieldBorderSumTransaction is a SAMRAI Transaction created by the - * FieldBorderSumTransactionFactory provided (via createShedule) to schedules that accumulate - * incomplete density and flux on ghost box overlaps. + * A FieldBorderOpTransaction is a SAMRAI Transaction created by the + * FieldBorderOpTransactionFactory provided (via createShedule) to schedules that executes + * the provided operation on ghost box overlaps. * - * Due to the lack of neighbor particle contributions, some domain nodes and ghost nodes - * have incomplete moments after deposition. The complement of these nodes is what has - * been deposited on (also incomplete) neighbor nodes. + * One context this is used for is due to the lack of neighbor particle contributions, + * some domain nodes and ghost nodes have incomplete moments after deposition. + * The complement of these nodes is what has been deposited on (also incomplete) neighbor nodes. * * Default SAMRAI transaction calls PatchData::copy and PatchData::packStream * This transaction defines these override to these methods to call specific methods - * of FieldData to perform the += instead of =. - * These methods are copyAndSum and unpackStreamAndSum. - * + * of FieldData to perform arbitrary operations. */ -template -class FieldBorderSumTransaction : public SAMRAI::tbox::Transaction +template +class FieldBorderOpTransaction : public SAMRAI::tbox::Transaction { public: - FieldBorderSumTransaction(std::shared_ptr const& dst_level, - std::shared_ptr const& src_level, - std::shared_ptr const& overlap, - SAMRAI::hier::Box const& dst_node, SAMRAI::hier::Box const& src_node, - SAMRAI::xfer::RefineClasses::Data const** refine_data, int item_id) + FieldBorderOpTransaction(std::shared_ptr const& dst_level, + std::shared_ptr const& src_level, + std::shared_ptr const& overlap, + SAMRAI::hier::Box const& dst_node, SAMRAI::hier::Box const& src_node, + SAMRAI::xfer::RefineClasses::Data const** refine_data, int item_id) : d_dst_level(dst_level) , d_src_level(src_level) , d_overlap(overlap) @@ -60,7 +59,7 @@ class FieldBorderSumTransaction : public SAMRAI::tbox::Transaction TBOX_ASSERT_OBJDIM_EQUALITY4(*dst_level, *src_level, dst_node, src_node); } - virtual ~FieldBorderSumTransaction() {} + virtual ~FieldBorderOpTransaction() {} virtual bool canEstimateIncomingMessageSize(); @@ -94,10 +93,10 @@ class FieldBorderSumTransaction : public SAMRAI::tbox::Transaction }; -template -bool FieldBorderSumTransaction::canEstimateIncomingMessageSize() +template +bool FieldBorderOpTransaction::canEstimateIncomingMessageSize() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::canEstimateIncomingMessageSize"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::canEstimateIncomingMessageSize"); bool can_estimate = false; if (getSourceProcessor() == d_src_level->getBoxLevel()->getMPI().getRank()) { @@ -115,74 +114,78 @@ bool FieldBorderSumTransaction::canEstimateIncomingMessageSize() } -template -size_t FieldBorderSumTransaction::computeIncomingMessageSize() +template +size_t FieldBorderOpTransaction::computeIncomingMessageSize() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::computeIncomingMessageSize"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::computeIncomingMessageSize"); d_incoming_bytes = d_dst_level->getPatch(d_dst_node.getGlobalId()) ->getPatchData(d_refine_data[d_item_id]->d_scratch) ->getDataStreamSize(*d_overlap); return d_incoming_bytes; } -template -size_t FieldBorderSumTransaction::computeOutgoingMessageSize() +template +size_t FieldBorderOpTransaction::computeOutgoingMessageSize() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::computeOutgoingMessageSize"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::computeOutgoingMessageSize"); d_outgoing_bytes = d_src_level->getPatch(d_src_node.getGlobalId()) ->getPatchData(d_refine_data[d_item_id]->d_src) ->getDataStreamSize(*d_overlap); return d_outgoing_bytes; } -template -int FieldBorderSumTransaction::getSourceProcessor() +template +int FieldBorderOpTransaction::getSourceProcessor() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::getSourceProcessor"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::getSourceProcessor"); return d_src_node.getOwnerRank(); } -template -int FieldBorderSumTransaction::getDestinationProcessor() +template +int FieldBorderOpTransaction::getDestinationProcessor() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::getDestinationProcessor"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::getDestinationProcessor"); return d_dst_node.getOwnerRank(); } -template -void FieldBorderSumTransaction::packStream(SAMRAI::tbox::MessageStream& stream) +template +void FieldBorderOpTransaction::packStream( + SAMRAI::tbox::MessageStream& stream) { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::packStream"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::packStream"); d_src_level->getPatch(d_src_node.getGlobalId()) ->getPatchData(d_refine_data[d_item_id]->d_src) ->packStream(stream, *d_overlap); } -template -void FieldBorderSumTransaction::unpackStream(SAMRAI::tbox::MessageStream& stream) +template +void FieldBorderOpTransaction::unpackStream( + SAMRAI::tbox::MessageStream& stream) { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::unpackStream"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::unpackStream"); std::shared_ptr onode_dst_data( SAMRAI_SHARED_PTR_CAST( d_dst_level->getPatch(d_dst_node.getGlobalId()) ->getPatchData(d_refine_data[d_item_id]->d_scratch))); TBOX_ASSERT(onode_dst_data); - onode_dst_data->unpackStreamAndSum(stream, *d_overlap); + onode_dst_data->template unpackStreamAnd(stream, *d_overlap); } -template -void FieldBorderSumTransaction::printClassData(std::ostream& stream) const +template +void FieldBorderOpTransaction::printClassData(std::ostream& stream) const { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::printClassData"); - throw std::runtime_error("FieldBorderSumTransaction::printClassData!"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::printClassData"); + + throw std::runtime_error("FieldBorderOpTransaction::printClassData!"); } -template -void FieldBorderSumTransaction::copyLocalData() +template +void FieldBorderOpTransaction::copyLocalData() { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransaction::copyLocalData"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransaction::copyLocalData"); + std::shared_ptr onode_dst_data( SAMRAI_SHARED_PTR_CAST( d_dst_level->getPatch(d_dst_node.getGlobalId()) @@ -195,12 +198,12 @@ void FieldBorderSumTransaction::copyLocalData() ->getPatchData(d_refine_data[d_item_id]->d_src))); TBOX_ASSERT(onode_src_data); - onode_dst_data->sum(*onode_src_data, *d_overlap); + onode_dst_data->template operate(*onode_src_data, *d_overlap); } -template -class FieldBorderSumTransactionFactory : public SAMRAI::xfer::RefineTransactionFactory +template +class FieldBorderOpTransactionFactory : public SAMRAI::xfer::RefineTransactionFactory { public: std::shared_ptr @@ -222,8 +225,9 @@ class FieldBorderSumTransactionFactory : public SAMRAI::xfer::RefineTransactionF TBOX_ASSERT(refine_data != 0); TBOX_ASSERT_OBJDIM_EQUALITY4(*dst_level, *src_level, dst_node, src_node); - PHARE_LOG_SCOPE(2, "FieldBorderSumTransactionFactory::allocate"); - return std::make_shared>( + PHARE_LOG_SCOPE(2, "FieldBorderOpTransactionFactory::allocate"); + + return std::make_shared>( dst_level, src_level, overlap, dst_node, src_node, refine_data, item_id); } @@ -231,7 +235,7 @@ class FieldBorderSumTransactionFactory : public SAMRAI::xfer::RefineTransactionF preprocessScratchSpace(std::shared_ptr const& level, double fill_time, SAMRAI::hier::ComponentSelector const& preprocess_vector) const override { - PHARE_LOG_SCOPE(2, "FieldBorderSumTransactionFactory::preprocessScratchSpace"); + PHARE_LOG_SCOPE(2, "FieldBorderOpTransactionFactory::preprocessScratchSpace"); // noop } @@ -239,4 +243,4 @@ class FieldBorderSumTransactionFactory : public SAMRAI::xfer::RefineTransactionF } // namespace PHARE::amr -#endif // PHARE_AMR_MESSENGERS_FIELD_SUM_TRANSACTION_HPP +#endif // PHARE_AMR_MESSENGERS_FIELD_OP_TRANSACTION_HPP diff --git a/src/amr/messengers/hybrid_hybrid_messenger_strategy.hpp b/src/amr/messengers/hybrid_hybrid_messenger_strategy.hpp index 4efc060ae..885b0555d 100644 --- a/src/amr/messengers/hybrid_hybrid_messenger_strategy.hpp +++ b/src/amr/messengers/hybrid_hybrid_messenger_strategy.hpp @@ -4,9 +4,9 @@ #include "core/def.hpp" // IWYU pragma: keep #include "core/logger.hpp" #include "core/def/phare_mpi.hpp" // IWYU pragma: keep +#include "core/utilities/types.hpp" #include "core/hybrid/hybrid_quantities.hpp" #include "core/numerics/interpolator/interpolator.hpp" -#include "core/utilities/types.hpp" #include "refiner_pool.hpp" #include "synchronizer_pool.hpp" @@ -247,6 +247,11 @@ namespace amr for (auto& refiner : popDensityBorderSumRefiners_) refiner.registerLevel(hierarchy, level); + for (auto& refiner : ionFluxBorderMaxRefiners_) + refiner.registerLevel(hierarchy, level); + for (auto& refiner : ionDensityBorderMaxRefiners_) + refiner.registerLevel(hierarchy, level); + // root level is not initialized with a schedule using coarser level data // so we don't create these schedules if root level // TODO this 'if' may not be OK if L0 is regrided @@ -448,6 +453,8 @@ namespace amr { using value_type = FieldT::value_type; + assert(popDensityBorderSumRefiners_.size() % ions.size() == 0); + std::size_t const fieldsPerPop = popDensityBorderSumRefiners_.size() / ions.size(); for (std::size_t i = 0; i < ions.size(); ++i) @@ -480,6 +487,17 @@ namespace amr } + void fillIonBorders(IonsT& /*ions*/, level_t& level, double const fillTime) override + { + assert(ionFluxBorderMaxRefiners_.size() == 1); + assert(ionDensityBorderMaxRefiners_.size() == 2); + + for (auto& refiner : ionFluxBorderMaxRefiners_) + refiner.fill(level.getLevelNumber(), fillTime); + for (auto& refiner : ionDensityBorderMaxRefiners_) + refiner.fill(level.getLevelNumber(), fillTime); + } + /** @@ -844,19 +862,33 @@ namespace amr for (auto const& vecfield : info->ghostFlux) - { popFluxBorderSumRefiners_.emplace_back(resourcesManager_) .addStaticRefiner( sumVec_.name(), vecfield, nullptr, sumVec_.name(), std::make_shared< TensorFieldGhostInterpOverlapFillPattern>()); - } for (auto const& field : info->sumBorderFields) popDensityBorderSumRefiners_.emplace_back(resourcesManager_) .addStaticRefiner( sumField_.name(), field, nullptr, sumField_.name(), std::make_shared>()); + + + assert(info->maxBorderFields.size() == 2); // mass & charge densities + for (auto const& field : info->maxBorderFields) + ionDensityBorderMaxRefiners_.emplace_back(resourcesManager_) + .addStaticRefiner( + field, field, nullptr, field, + std::make_shared>()); + + assert(info->maxBorderVecFields.size() == 1); + for (auto const& vecfield : info->maxBorderVecFields) + ionFluxBorderMaxRefiners_.emplace_back(resourcesManager_) + .addStaticRefiner( + vecfield, vecfield, nullptr, vecfield, + std::make_shared< + TensorFieldGhostInterpOverlapFillPattern>()); } @@ -1007,6 +1039,8 @@ namespace amr using PatchGhostRefinerPool = RefinerPool; using FieldGhostSumRefinerPool = RefinerPool; using VecFieldGhostSumRefinerPool = RefinerPool; + using FieldGhostMaxRefinerPool = RefinerPool; + using VecFieldGhostMaxRefinerPool = RefinerPool; using FieldFillPattern_t = FieldFillPattern; using TensorFieldFillPattern_t = TensorFieldFillPattern; @@ -1015,6 +1049,9 @@ namespace amr //! += density on ghost box overlap incomplete population moment nodes std::vector popDensityBorderSumRefiners_; + std::vector ionDensityBorderMaxRefiners_; + std::vector ionFluxBorderMaxRefiners_; + InitRefinerPool electricInitRefiners_{resourcesManager_}; diff --git a/src/amr/messengers/hybrid_messenger.hpp b/src/amr/messengers/hybrid_messenger.hpp index 9b86dc7c0..1153f1720 100644 --- a/src/amr/messengers/hybrid_messenger.hpp +++ b/src/amr/messengers/hybrid_messenger.hpp @@ -356,6 +356,11 @@ namespace amr strat_->fillDensityBorders(ions, level, fillTime); } + void fillIonBorders(IonsT& ions, SAMRAI::hier::PatchLevel& level, double const fillTime) + { + strat_->fillIonBorders(ions, level, fillTime); + } + /* ------------------------------------------------------------------------- End HybridMessenger Interface -------------------------------------------------------------------------*/ diff --git a/src/amr/messengers/hybrid_messenger_info.hpp b/src/amr/messengers/hybrid_messenger_info.hpp index ef3b984fa..27c47a5a6 100644 --- a/src/amr/messengers/hybrid_messenger_info.hpp +++ b/src/amr/messengers/hybrid_messenger_info.hpp @@ -66,6 +66,10 @@ namespace amr std::vector ghostCurrent; std::vector ghostBulkVelocity; + + std::vector maxBorderFields; + std::vector maxBorderVecFields; + // below are the descriptions of the electric field that we use in the refluxing std::string refluxElectric; std::string fluxSumElectric; diff --git a/src/amr/messengers/hybrid_messenger_strategy.hpp b/src/amr/messengers/hybrid_messenger_strategy.hpp index 447068da2..a89ec9cb9 100644 --- a/src/amr/messengers/hybrid_messenger_strategy.hpp +++ b/src/amr/messengers/hybrid_messenger_strategy.hpp @@ -134,6 +134,9 @@ namespace amr virtual void fillDensityBorders(IonsT& ions, SAMRAI::hier::PatchLevel& level, double const fillTime) = 0; + virtual void fillIonBorders(IonsT& ions, SAMRAI::hier::PatchLevel& level, + double const fillTime) + = 0; std::string name() const { return stratname_; } diff --git a/src/amr/messengers/mhd_hybrid_messenger_strategy.hpp b/src/amr/messengers/mhd_hybrid_messenger_strategy.hpp index cc82defe0..a91896936 100644 --- a/src/amr/messengers/mhd_hybrid_messenger_strategy.hpp +++ b/src/amr/messengers/mhd_hybrid_messenger_strategy.hpp @@ -123,6 +123,12 @@ namespace amr double const /*fillTime*/) override { } + void fillIonBorders(IonsT& /*ions*/, SAMRAI::hier::PatchLevel& /*level*/, + double const /*fillTime*/) override + { + } + + void firstStep(IPhysicalModel& /*model*/, SAMRAI::hier::PatchLevel& /*level*/, std::shared_ptr const& /*hierarchy*/, diff --git a/src/amr/messengers/refiner.hpp b/src/amr/messengers/refiner.hpp index 5240058e3..d6d1a831f 100644 --- a/src/amr/messengers/refiner.hpp +++ b/src/amr/messengers/refiner.hpp @@ -2,11 +2,10 @@ #define PHARE_REFINER_HPP #include "communicator.hpp" -#include "core/data/vecfield/vecfield.hpp" -#include "amr/messengers/field_sum_transaction.hpp" +#include "amr/messengers/field_operate_transaction.hpp" +#include "core/utilities/types.hpp" -#include #include @@ -23,6 +22,8 @@ enum class RefinerType { PatchFieldBorderSum, PatchVecFieldBorderSum, PatchTensorFieldBorderSum, + PatchFieldBorderMax, + PatchVecFieldBorderMax, ExteriorGhostParticles }; @@ -33,6 +34,9 @@ class Refiner : private Communicator { using FieldData_t = ResourcesManager::UserField_t::patch_data_type; + using SetMaxOp = core::SetMax; + using PlusEqualsOp = core::PlusEquals; + // hard coded rank cause there's no real tensorfields that use this code yet using TensorFieldData_t = ResourcesManager::template UserTensorField_t<2>::patch_data_type; using VecFieldData_t = ResourcesManager::template UserTensorField_t<1>::patch_data_type; @@ -73,7 +77,7 @@ class Refiner : private Communicator levelNumber); } - if constexpr (Type == RefinerType::PatchGhostField) + else if constexpr (Type == RefinerType::PatchGhostField) { this->add(algo, algo->createSchedule(level, patchStrat_.get()), levelNumber); } @@ -86,17 +90,20 @@ class Refiner : private Communicator this->add(algo, algo->createSchedule( level, patchStrat_.get(), - std::make_shared>()), + std::make_shared< + FieldBorderOpTransactionFactory>()), levelNumber); } + else if constexpr (Type == RefinerType::PatchTensorFieldBorderSum) { this->add( algo, algo->createSchedule( level, patchStrat_.get(), - std::make_shared>()), + std::make_shared< + FieldBorderOpTransactionFactory>()), levelNumber); } @@ -106,7 +113,34 @@ class Refiner : private Communicator this->add(algo, algo->createSchedule( level, patchStrat_.get(), - std::make_shared>()), + std::make_shared< + FieldBorderOpTransactionFactory>()), + levelNumber); + } + + + // schedule used to == max of density and flux for populations + // on complete overlaped ghost box nodes + else if constexpr (Type == RefinerType::PatchFieldBorderMax) + { + this->add( + algo, + algo->createSchedule( + level, patchStrat_.get(), + std::make_shared>()), + levelNumber); + } + + + // schedule used to == max of density and flux for populations + // on complete overlaped ghost box nodes + else if constexpr (Type == RefinerType::PatchVecFieldBorderMax) + { + this->add(algo, + algo->createSchedule( + level, patchStrat_.get(), + std::make_shared< + FieldBorderOpTransactionFactory>()), levelNumber); } @@ -175,6 +209,11 @@ class Refiner : private Communicator { this->add(algo, algo->createSchedule(level, patchStrat_.get()), levelNumber); } + + else + { + throw std::runtime_error("No Schedule for RefinerType"); + } } } diff --git a/src/amr/physical_models/hybrid_model.hpp b/src/amr/physical_models/hybrid_model.hpp index ecdcf1c9a..08dfcc878 100644 --- a/src/amr/physical_models/hybrid_model.hpp +++ b/src/amr/physical_models/hybrid_model.hpp @@ -179,6 +179,10 @@ void HybridModel::f hybridInfo.sumBorderFields.emplace_back(pop.particleDensity().name()); hybridInfo.sumBorderFields.emplace_back(pop.chargeDensity().name()); } + + hybridInfo.maxBorderFields.emplace_back(state.ions.massDensity().name()); + hybridInfo.maxBorderFields.emplace_back(state.ions.chargeDensity().name()); + hybridInfo.maxBorderVecFields.emplace_back(state.ions.velocity().name()); } diff --git a/src/amr/solvers/solver_ppc.hpp b/src/amr/solvers/solver_ppc.hpp index 7987e0d94..2779052fe 100644 --- a/src/amr/solvers/solver_ppc.hpp +++ b/src/amr/solvers/solver_ppc.hpp @@ -580,6 +580,8 @@ void SolverPPC::moveIons_(level_t& level, ModelViews_t& for (auto& state : views) ionUpdater_.updateIons(state.ions); + fromCoarser.fillIonBorders(views.model().state.ions, level, newTime); + // no need to update time, since it has been done before // now Ni and Vi are calculated we can fill pure ghost nodes // these were not completed by the deposition of patch and levelghost particles diff --git a/src/core/utilities/mpi_utils.hpp b/src/core/utilities/mpi_utils.hpp index b327d8bca..c093e34d3 100644 --- a/src/core/utilities/mpi_utils.hpp +++ b/src/core/utilities/mpi_utils.hpp @@ -159,7 +159,10 @@ NO_DISCARD std::vector collectVector(Vector const& sendBuff, int mpi_siz std::vector collected; for (int i = 0; i < mpi_size; i++) { - collected.emplace_back(&rcvBuff[offset], &rcvBuff[offset] + perMPISize[i]); + if (perMPISize[i] == 0) + collected.emplace_back(); + else + collected.emplace_back(&rcvBuff[offset], &rcvBuff[offset] + perMPISize[i]); offset += perMPISize[i]; } return collected; diff --git a/src/core/utilities/types.hpp b/src/core/utilities/types.hpp index 7ead8b862..4b9108666 100644 --- a/src/core/utilities/types.hpp +++ b/src/core/utilities/types.hpp @@ -560,6 +560,13 @@ struct PlusEquals D& d; }; +template +struct SetMax +{ + void operator()(auto& d0) { d = std::max(d, d0); } + D& d; +}; + } // namespace PHARE::core diff --git a/src/diagnostic/diagnostic_model_view.hpp b/src/diagnostic/diagnostic_model_view.hpp index 4885db24d..8db43b3ff 100644 --- a/src/diagnostic/diagnostic_model_view.hpp +++ b/src/diagnostic/diagnostic_model_view.hpp @@ -6,7 +6,7 @@ #include "amr/physical_models/mhd_model.hpp" #include "amr/physical_models/hybrid_model.hpp" -#include "amr/messengers/field_sum_transaction.hpp" +#include "amr/messengers/field_operate_transaction.hpp" #include "amr/data/field/field_variable_fill_pattern.hpp" #include "dict.hpp" @@ -170,12 +170,14 @@ class BaseModelView : public IModelView { auto& getOrCreateSchedule(auto& hierarchy, int const ilvl) { + using PlusEqualsOp = core::PlusEquals; if (not MTschedules.count(ilvl)) MTschedules.try_emplace( - ilvl, MTalgo->createSchedule( - hierarchy.getPatchLevel(ilvl), 0, - std::make_shared< - amr::FieldBorderSumTransactionFactory>())); + ilvl, + MTalgo->createSchedule( + hierarchy.getPatchLevel(ilvl), 0, + std::make_shared>())); return *MTschedules[ilvl]; } diff --git a/tests/simulator/__init__.py b/tests/simulator/__init__.py index 0876267a2..47aa1f8b7 100644 --- a/tests/simulator/__init__.py +++ b/tests/simulator/__init__.py @@ -41,7 +41,7 @@ def basicSimulatorArgs(dim: int, interp: int, **kwargs): _, smallest_patch_size = check_patch_size(dim, interp_order=interp, cells=cells) dl = [1.0 / v for v in cells] - b0 = [[3] * dim, [8] * dim] + b0 = [[3] * dim, [12] * dim] args = { "interp_order": interp, "smallest_patch_size": smallest_patch_size,