Prevent OS core dump creation for intentionally crashing tests

src/tests/Common/CoreCLRTestLibrary/Utilities.cs

@@ -488,5 +488,37 @@ public static void ExecuteAndUnload(string assemblyPath, string typeName, string
 
             Assert.False(alcWeakRef.IsAlive);
         }
+
+        [StructLayout(LayoutKind.Sequential)]
+        public struct RLimit
+        {
+            public ulong rlim_cur; // Soft limit
+            public ulong rlim_max; // Hard limit
+        }
+
+        public const int RLIMIT_CORE = 4; // Core file size
+
+        [DllImport("libc", SetLastError = true)]
+        public static extern int setrlimit(int resource, ref RLimit rlim);
+
+        // Ensure that the OS doesn't generate core dump for the current process
+        public static void DisableOSCoreDump()
+        {
+            // At present, RLimit is defined in a way where the fields are always 64-bit.
+            // Before adding support for a new platform, its definition of rlimit should be confirmed.
+            if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
+            {
+                RLimit rlimit = new RLimit
+                {
+                    rlim_cur = 0,
+                    rlim_max = 0
+                };
+
+                if (setrlimit(RLIMIT_CORE, ref rlimit) != 0)
+                {
+                    throw new Exception($"Failed to disable core dump, error {Marshal.GetLastPInvokeError()} - {Marshal.GetLastPInvokeErrorMessage()}.");
+                }
+            }
+        }
     }
 }

src/tests/baseservices/exceptions/simple/ParallelCrash.cs

@@ -7,6 +7,7 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Threading;
+using TestLibrary;
 
 // Runtime stability in the presence of concurrent fatal errors
 
@@ -20,6 +21,9 @@ public class ParallelCrash
 
     public static int Main(string[] args)
     {
+        // Ensure that the OS doesn't generate core dump for this intentionally crashing process
+        Utilities.DisableOSCoreDump();
+
         s_crashMainThread = true;
         s_crashWorkerThreads = true;
         if (args.Length > 0)

src/tests/baseservices/exceptions/simple/ParallelCrash.csproj

@@ -8,4 +8,7 @@
   <ItemGroup>
     <Compile Include="ParallelCrash.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+  </ItemGroup>
 </Project>

src/tests/baseservices/exceptions/stackoverflow/stackoverflow.cs

@@ -3,6 +3,8 @@
 using System;
 using System.Threading;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using TestLibrary;
 
 namespace TestStackOverflow
 {
@@ -136,6 +138,9 @@ static void SecondaryThreadsTest(bool smallframe)
 
         static void Main(string[] args)
         {
+            // Ensure that the OS doesn't generate core dump for this intentionally crashing process
+            Utilities.DisableOSCoreDump();
+
             bool smallframe = (args[0] == "smallframe");
             if (args[1] == "secondary")
             {

src/tests/baseservices/exceptions/stackoverflow/stackoverflow.csproj

@@ -10,5 +10,8 @@
   <ItemGroup>
     <Compile Include="stackoverflow.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+  </ItemGroup>
 </Project>
 

src/tests/baseservices/exceptions/stackoverflow/stackoverflow3.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 using System;
+using System.Runtime.InteropServices;
+using TestLibrary;
 
 namespace TestStackOverflow3
 {
@@ -11,6 +13,9 @@ class Program
 
         public static void Main()
         {
+            // Ensure that the OS doesn't generate core dump for this intentionally crashing process
+            Utilities.DisableOSCoreDump();
+
             Program ex = new Program();
             ex.Execute();
         }

src/tests/baseservices/exceptions/stackoverflow/stackoverflow3.csproj

@@ -11,5 +11,8 @@
   <ItemGroup>
     <Compile Include="stackoverflow3.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+  </ItemGroup>
 </Project>
 

src/tests/baseservices/exceptions/unhandled/unhandled.cs

@@ -6,6 +6,7 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Threading;
+using TestLibrary;
 
 namespace TestUnhandledException
 {
@@ -40,6 +41,9 @@ private static void SetDllResolver()
 
         static void Main(string[] args)
         {
+            // Ensure that the OS doesn't generate core dump for this intentionally crashing process
+            Utilities.DisableOSCoreDump();
+
             if (args[0] == "main")
             {
                 throw new Exception("Test");