test(rccl): suppress core dumps in expected-crash tests

projects/rccl/test/host/init-test.cc

@@ -21,6 +21,10 @@
 #include <memory>
 #include <new>
 #include <string>
+#if defined(__linux__)
+#include <sys/prctl.h>
+#endif
+#include <sys/resource.h>
 #include <utility>
 #include <vector>
 
@@ -181,6 +185,26 @@ class DeleteWatch {
 #define DEATH_BY_SEGV ::testing::KilledBySignal(SIGSEGV)
 #endif
 
+// These tests deliberately raise fatal signals. Some CI environments enable core dumps, and a
+// shuffled death test can inherit a large address space from an earlier test. Dumping that address
+// space turns a millisecond assertion into a multi-second test. PR_SET_DUMPABLE covers both regular
+// core files and piped collectors (for which Linux ignores RLIMIT_CORE); retain the limit as defense
+// in depth. Apply both inside the death statement so only gtest's child is affected and unexpected
+// parent-process crashes remain dumpable.
+static void DisableCoreDumpsForExpectedCrash() {
+#if defined(__linux__)
+  if (prctl(PR_SET_DUMPABLE, 0) != 0) {
+    std::perror("prctl(PR_SET_DUMPABLE) failed");
+    _exit(1);
+  }
+#endif
+  const struct rlimit noCore = {0, 0};
+  if (setrlimit(RLIMIT_CORE, &noCore) != 0) {
+    std::perror("setrlimit(RLIMIT_CORE) failed");
+    _exit(1);
+  }
+}
+
 class InitMicrotest : public ::testing::Test {
  protected:
   void SetUp() override {
@@ -819,7 +843,12 @@ TEST_F(InitMicrotest, P2pSchedule_ZeroGroupSizeParam_DiesOnDivideByZero) {
   P2pScheduleComm c(/*nNodes=*/2, /*node=*/0, /*localRank=*/0, /*nRanks=*/8,
                     /*maxLocalRanks=*/4, {4, 4});
   // Pin the signal: EXPECT_DEATH("") would also accept ::abort(), _exit(1) or a null deref at the same spot.
-  EXPECT_EXIT(ncclP2pSchedule(c.get()), ::testing::KilledBySignal(SIGFPE), "");
+  EXPECT_EXIT(
+      {
+        DisableCoreDumpsForExpectedCrash();
+        (void)ncclP2pSchedule(c.get());
+      },
+      ::testing::KilledBySignal(SIGFPE), "");
 }
 
 TEST_F(InitMicrotest, P2pSchedule_SingleNode_BuildsFullSchedule) {
@@ -1226,9 +1255,13 @@ TEST_F(InitMicrotest, CommShrink_NullNewcomm_DiesOnNullDeref) {
   ReadyComm rc;
   int exclude[1] = {0};
   // Match the message too: the signal alone would also accept a crash arriving BEFORE the newcomm validation.
-  EXPECT_EXIT(ncclCommShrink_impl(rc.get(), exclude, /*excludeRanksCount=*/1, nullptr,
-                                  /*config=*/nullptr, /*shrinkFlags=*/0),
-              DEATH_BY_SEGV, "newcomm argument is NULL");
+  EXPECT_EXIT(
+      {
+        DisableCoreDumpsForExpectedCrash();
+        (void)ncclCommShrink_impl(rc.get(), exclude, /*excludeRanksCount=*/1, nullptr,
+                                  /*config=*/nullptr, /*shrinkFlags=*/0);
+      },
+      DEATH_BY_SEGV, "newcomm argument is NULL");
 }
 
 // Full strings, not substrings: a prefix check cannot see a changed NCCL_DEBUG level or a dropped "(run with)" hint.
@@ -3687,6 +3720,7 @@ TEST_F(InitMicrotest, SetCommAbortFlags_LargePositiveValue_StoredVerbatim) {
 TEST_F(InitMicrotest, SetCommAbortFlags_NullChildDevUnderNonNullChildFlag_DiesOnNullDeref) {
   EXPECT_EXIT(
       {
+        DisableCoreDumpsForExpectedCrash();
         AbortFlagsComm c;
         c.get()->childAbortFlagDev = nullptr;
         fprintf(stderr, "setCommAbortFlags-reached-with-null-childAbortFlagDev\n");