Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions projects/rccl/src/transport/net_ib_cast/common_cast.h
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ struct alignas(64) ncclIbDev {
char* pciPath;
int realPort;
int maxQp;
int maxCqe;
float latency;
struct ncclIbMrCache mrCache;
int ar; // ADAPTIVE_ROUTING
Expand Down
6 changes: 6 additions & 0 deletions projects/rccl/src/transport/net_ib_cast/connect.cc
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,9 @@ ncclResult_t IbCastConnect(void* ctx, int dev, void* opaqueHandle, void** sendCo
if (comm->base.resiliency) {
IbCastResiliencyDataCqSizeGet(comm->base.resiliency, i, &cqSize);
}
if (IbCastDevs[ibDevN].maxCqe > 0) {
cqSize = std::min(IbCastDevs[ibDevN].maxCqe, cqSize);
}
NCCLCHECKGOTO(IbCastInitCommDevBase(ibDevN, &comm->devs[i].base, &comm->base.stats, cqSize), ret, fail);
comm->ar = comm->ar && IbCastDevs[ibDevN].ar; // ADAPTIVE_ROUTING - if all merged devs have it enabled
if (comm->base.resiliency) {
Expand Down Expand Up @@ -1492,6 +1495,9 @@ ncclResult_t IbCastAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle
if (rComm->base.resiliency) {
IbCastResiliencyDataCqSizeGet(rComm->base.resiliency, i, &cqSize);
}
if (IbCastDevs[ibDevN].maxCqe > 0) {
cqSize = std::min(IbCastDevs[ibDevN].maxCqe, cqSize);
}
NCCLCHECKGOTO(IbCastInitCommDevBase(ibDevN, &rCommDev->base, &rComm->base.stats, cqSize), ret, fail);
if (rComm->base.resiliency) {
NCCLCHECKGOTO(IbCastResiliencyDevInit(rComm->base.resiliency, i, &IbCastDevs[ibDevN]), ret, fail);
Expand Down
1 change: 1 addition & 0 deletions projects/rccl/src/transport/net_ib_cast/init.cc
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,7 @@ ncclResult_t IbCastInitDevices(ncclDebugLogger_t logFunction, ncclProfilerCallba
}

IbCastDevs[IbCastNDevs].maxQp = devAttr.max_qp;
IbCastDevs[IbCastNDevs].maxCqe = devAttr.max_cqe;
IbCastDevs[IbCastNDevs].oooRqSize = oooRqSize;
IbCastDevs[IbCastNDevs].mrCache.capacity = 0;
IbCastDevs[IbCastNDevs].mrCache.population = 0;
Expand Down
42 changes: 28 additions & 14 deletions projects/rccl/test/common/MPIEnvironment.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#ifdef MPI_TESTS_ENABLED

#include <chrono>
#include <cstdlib>
#include <thread>

/**
Expand Down Expand Up @@ -237,12 +238,22 @@ void MPIEnvironment::TearDown()
if(barrier_result == MPI_SUCCESS)
{
// Wait for barrier with a timeout.
// 3 s: ncclCommDestroy is now explicitly called (and MPI-barrier'd) inside
// cleanupTestCommunicator before TearDown runs, so the only remaining rank
// skew here is log-file I/O (RCCL_MPI_LOG_ALL_RANKS) which is fast.
int flag = 0;
auto timeout_start = std::chrono::steady_clock::now();
const auto timeout_duration = std::chrono::seconds(3);
// ncclCommDestroy is now explicitly called (and MPI-barrier'd) inside
// cleanupTestCommunicator before TearDown runs, so the common remaining rank
// skew here is log-file I/O (RCCL_MPI_LOG_ALL_RANKS). However, heavy
// multi-phase tests (e.g. GrowMPITest, NetIbMPITest CtsDepthStress) can have
// ranks reach TearDown more than a few seconds apart even on success, so a
// 3 s timeout produced false "out of sync" aborts. Use a more tolerant default
// and allow override via RCCL_TEST_TEARDOWN_TIMEOUT_SEC.
int flag = 0;
auto timeout_start = std::chrono::steady_clock::now();
int timeout_sec = 60;
if(const char* env = std::getenv("RCCL_TEST_TEARDOWN_TIMEOUT_SEC"))
{
int v = std::atoi(env);
if(v > 0) timeout_sec = v;
Comment on lines +250 to +254
}
const auto timeout_duration = std::chrono::seconds(timeout_sec);

while(!flag)
{
Expand All @@ -254,18 +265,21 @@ void MPIEnvironment::TearDown()
auto elapsed = std::chrono::steady_clock::now() - timeout_start;
if(elapsed > timeout_duration)
{
// Timeout - ranks are out of sync!
// Timeout - ranks are genuinely out of sync (e.g. a rank died or
// deadlocked in the test body). Force abort.
std::fprintf(
stderr,
"Rank %d: TIMEOUT in TearDown barrier - ranks out of sync, forcing abort\n",
world_rank);
"Rank %d: TIMEOUT (%ds) in TearDown barrier - ranks out of sync, forcing abort\n",
world_rank,
timeout_sec);
std::fflush(stderr);

// Cancel the barrier request
MPI_Cancel(&barrier_req);
MPI_Request_free(&barrier_req);

// Force abort - can't safely continue
// NOTE: Do NOT MPI_Cancel(&barrier_req) here. Cancelling a
// nonblocking *collective* request (MPI_Ibarrier) is forbidden by
// the MPI standard; OpenMPI returns MPI_ERR_REQUEST, and because
// MPI_COMM_WORLD uses MPI_ERRORS_ARE_FATAL that call aborts the job
// itself with a misleading error before we reach MPI_Abort. Since
// MPI_Abort tears the whole job down anyway, just call it directly.
MPI_Abort(MPI_COMM_WORLD, 1);
return;
}
Expand Down
Loading