Skip to content

Commit 4835c73

Browse files
gberg617gberg617
committed
added non-blocking root communicator
1 parent 56c8787 commit 4835c73

File tree

4 files changed

+275
-3
lines changed

4 files changed

+275
-3
lines changed

‎src/axom/lumberjack/MPIUtility.cpp

+35-2
Original file line numberDiff line numberDiff line change
@@ -49,16 +49,49 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm)
4949
return charArray;
5050
}
5151

52+
const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag)
53+
{
54+
const int mpiTag = (tag == false) ? LJ_TAG : tag;
55+
char* charArray = nullptr;
56+
int messageSize = -1;
57+
MPI_Status mpiStatus;
58+
59+
// Get size and source of MPI message
60+
int mpiFlag = true;
61+
MPI_Iprobe(MPI_ANY_SOURCE, tag, comm, &mpiFlag, &mpiStatus);
62+
63+
if (mpiFlag == true) {
64+
MPI_Get_count(&mpiStatus, MPI_CHAR, &messageSize);
65+
66+
// Setup where to receive the char array
67+
charArray = new char[messageSize + 1];
68+
charArray[messageSize] = '\0';
69+
70+
// Receive packed Message
71+
MPI_Recv(charArray,
72+
messageSize,
73+
MPI_CHAR,
74+
mpiStatus.MPI_SOURCE,
75+
mpiTag,
76+
comm,
77+
&mpiStatus);
78+
}
79+
80+
return charArray;
81+
}
82+
5283
void mpiNonBlockingSendMessages(MPI_Comm comm,
5384
int destinationRank,
54-
const char* packedMessagesToBeSent)
85+
const char* packedMessagesToBeSent,
86+
const int tag)
5587
{
88+
const int mpiTag = (tag == false) ? LJ_TAG : tag;
5689
MPI_Request mpiRequest;
5790
MPI_Isend(const_cast<char*>(packedMessagesToBeSent),
5891
strlen(packedMessagesToBeSent),
5992
MPI_CHAR,
6093
destinationRank,
61-
LJ_TAG,
94+
mpiTag,
6295
comm,
6396
&mpiRequest);
6497
MPI_Request_free(&mpiRequest);

‎src/axom/lumberjack/MPIUtility.hpp

+14-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@ namespace lumberjack
3030
*/
3131
const char* mpiBlockingReceiveMessages(MPI_Comm comm);
3232

33+
/*!
34+
*****************************************************************************
35+
* \brief Receives any Message sent to this rank, if there are any messages
36+
* that are sent. Returns null if no messages are sent.
37+
*
38+
* \param [in] comm The MPI Communicator.
39+
*****************************************************************************
40+
*/
41+
const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag = 0);
42+
3343
/*!
3444
*****************************************************************************
3545
* \brief Sends all Message sent to the given rank.
@@ -40,11 +50,14 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm);
4050
* \param [in] destinationRank Where the Message classes is being sent.
4151
* \param [in,out] packedMessagesToBeSent All of the Message classes to be sent
4252
* packed together.
53+
* \param [in] tag The MPI tag to use for communication. When set to zero,
54+
* MPI communication uses default LJ_Tag.
4355
*****************************************************************************
4456
*/
4557
void mpiNonBlockingSendMessages(MPI_Comm comm,
4658
int destinationRank,
47-
const char* packedMessagesToBeSent);
59+
const char* packedMessagesToBeSent,
60+
const int tag = 0);
4861
} // end namespace lumberjack
4962
} // end namespace axom
5063

‎src/axom/lumberjack/NonBlockingRootCommunicator.cpp

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
2+
// other Axom Project Developers. See the top-level LICENSE file for details.
3+
//
4+
// SPDX-License-Identifier: (BSD-3-Clause)
5+
6+
/*!
7+
******************************************************************************
8+
*
9+
* \file NonBlockingRootCommunicator.cpp
10+
*
11+
* \brief Implementation of the NonBlockingRootCommunicator class.
12+
*
13+
******************************************************************************
14+
*/
15+
16+
#include "axom/lumberjack/NonBlockingRootCommunicator.hpp"
17+
#include "axom/lumberjack/MPIUtility.hpp"
18+
19+
namespace axom
20+
{
21+
namespace lumberjack
22+
{
23+
void NonBlockingRootCommunicator::initialize(MPI_Comm comm, int ranksLimit)
24+
{
25+
m_mpiComm = comm;
26+
MPI_Comm_rank(m_mpiComm, &m_mpiCommRank);
27+
MPI_Comm_size(m_mpiComm, &m_mpiCommSize);
28+
m_ranksLimit = ranksLimit;
29+
}
30+
31+
void NonBlockingRootCommunicator::finalize() { }
32+
33+
int NonBlockingRootCommunicator::rank() { return m_mpiCommRank; }
34+
35+
void NonBlockingRootCommunicator::ranksLimit(int value) { m_ranksLimit = value; }
36+
37+
int NonBlockingRootCommunicator::ranksLimit() { return m_ranksLimit; }
38+
39+
int NonBlockingRootCommunicator::numPushesToFlush() { return 1; }
40+
41+
void NonBlockingRootCommunicator::push(const char* packedMessagesToBeSent,
42+
std::vector<const char*>& receivedPackedMessages)
43+
{
44+
constexpr int mpiTag = 32767;
45+
if(m_mpiCommRank == 0)
46+
{
47+
const char* currPackedMessages = nullptr;
48+
bool receive_messages = true;
49+
while(receive_messages)
50+
{
51+
currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm, mpiTag);
52+
53+
if(isPackedMessagesEmpty(currPackedMessages))
54+
{
55+
if (currPackedMessages == nullptr )
56+
{
57+
receive_messages = false;
58+
} else {
59+
delete [] currPackedMessages;
60+
}
61+
62+
}
63+
else
64+
{
65+
receivedPackedMessages.push_back(currPackedMessages);
66+
}
67+
68+
currPackedMessages = nullptr;
69+
70+
}
71+
}
72+
else
73+
{
74+
if(isPackedMessagesEmpty(packedMessagesToBeSent) == false)
75+
{
76+
mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent, mpiTag);
77+
}
78+
}
79+
}
80+
81+
bool NonBlockingRootCommunicator::isOutputNode()
82+
{
83+
if(m_mpiCommRank == 0)
84+
{
85+
return true;
86+
}
87+
return false;
88+
}
89+
90+
} // end namespace lumberjack
91+
} // end namespace axom

‎src/axom/lumberjack/NonBlockingRootCommunicator.hpp

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
2+
// other Axom Project Developers. See the top-level LICENSE file for details.
3+
//
4+
// SPDX-License-Identifier: (BSD-3-Clause)
5+
6+
/*!
7+
*******************************************************************************
8+
* \file NonBlockingRootCommunicator.hpp
9+
*
10+
* \brief This file contains the class definition of the
11+
* NonBlockingRootCommunicator.
12+
*******************************************************************************
13+
*/
14+
15+
#ifndef NONBLOCKINGROOTCOMMUNICATOR_HPP
16+
#define NONBLOCKINGROOTCOMMUNICATOR_HPP
17+
18+
#include "axom/lumberjack/Lumberjack.hpp"
19+
#include "axom/lumberjack/Communicator.hpp"
20+
21+
namespace axom
22+
{
23+
namespace lumberjack
24+
{
25+
/*!
26+
*******************************************************************************
27+
* \class NonBlockingRootCommunicator
28+
*
29+
* \brief Based off of RootCommunicator. This communicator pushes
30+
messages from any rank to root non-collectively, if any messages are sent.
31+
*******************************************************************************
32+
*/
33+
class NonBlockingRootCommunicator : public axom::lumberjack::Communicator
34+
{
35+
public:
36+
/*!
37+
*****************************************************************************
38+
* \brief Called to initialize the Communicator.
39+
*
40+
* This performs any setup work the Communicator needs before doing any work.
41+
* It is required that this is called before using the Communicator.
42+
*
43+
* \param [in] comm The MPI Communicator
44+
* \param [in] ranksLimit Limit on how many ranks are individually tracked per
45+
* Message.
46+
*****************************************************************************
47+
*/
48+
void initialize(MPI_Comm comm, int ranksLimit);
49+
50+
/*!
51+
*****************************************************************************
52+
* \brief Called to finalize the Communicator.
53+
*
54+
* This performs any cleanup work the Communicator needs to do before going
55+
* away.It is required that this is the last function called by the
56+
* Communicator.
57+
*****************************************************************************
58+
*/
59+
void finalize();
60+
61+
/*!
62+
*****************************************************************************
63+
* \brief Returns the MPI rank of this node
64+
*****************************************************************************
65+
*/
66+
int rank();
67+
68+
/*!
69+
*****************************************************************************
70+
* \brief Sets the rank limit.
71+
*
72+
* This is the limit on how many ranks generated a given message are
73+
* individually tracked per Message. After the limit has been reached, only
74+
* the Message::rankCount is incremented.
75+
*
76+
* \param [in] value Limits how many ranks are tracked per Message.
77+
*****************************************************************************
78+
*/
79+
void ranksLimit(int value);
80+
81+
/*!
82+
*****************************************************************************
83+
* \brief Returns the rank limit.
84+
*
85+
* This is the limit on how many ranks generated a given message are
86+
* individually tracked per Message. After the limit has been reached, only
87+
* the Message::rankCount is incremented.
88+
*****************************************************************************
89+
*/
90+
int ranksLimit();
91+
92+
/*!
93+
*****************************************************************************
94+
* \brief Function used by the Lumberjack class to indicate how many
95+
* individual pushes fully flush all currently held Message classes to the
96+
* root node. The Communicator class's tree structure dictates this.
97+
*****************************************************************************
98+
*/
99+
int numPushesToFlush();
100+
101+
/*!
102+
*****************************************************************************
103+
* \brief This pushes all messages to the root node.
104+
*
105+
* All messages are pushed to the root node. This is the same as
106+
* RootCommunicator::pushMessagesFully for this Communicator.
107+
*
108+
* \param [in] packedMessagesToBeSent All of this rank's Message classes
109+
* packed into a single buffer.
110+
* \param [in,out] receivedPackedMessages Received packed message buffers from
111+
* this nodes children.
112+
*****************************************************************************
113+
*/
114+
void push(const char* packedMessagesToBeSent,
115+
std::vector<const char*>& receivedPackedMessages);
116+
117+
/*!
118+
*****************************************************************************
119+
* \brief Function used by the Lumberjack to indicate whether this node should
120+
* be outputting messages. Only the root node outputs messages.
121+
*****************************************************************************
122+
*/
123+
bool isOutputNode();
124+
125+
private:
126+
MPI_Comm m_mpiComm;
127+
int m_mpiCommRank;
128+
int m_mpiCommSize;
129+
int m_ranksLimit;
130+
};
131+
132+
} // end namespace lumberjack
133+
} // end namespace axom
134+
135+
#endif

0 commit comments

Comments
 (0)