Add some tests for shared mutex corner cases

src/libraries/Common/src/Interop/Unix/Interop.GetPageSize.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.InteropServices;
+using System.Threading;
+
+internal static partial class Interop
+{
+    internal static partial class Sys
+    {
+        private static volatile int s_pageSize;
+
+        internal static int PageSize
+        {
+            get
+            {
+                int size = s_pageSize;
+                if (size == 0)
+                {
+                    Interlocked.CompareExchange(ref s_pageSize, GetPageSize(), 0);
+                    size = s_pageSize;
+                }
+
+                return size;
+            }
+        }
+
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_GetPageSize")]
+        private static partial int GetPageSize();
+    }
+}

src/libraries/System.Threading/tests/MutexTests.cs

@@ -250,6 +250,10 @@ public void MutualExclusionTest()
         public void Ctor_InvalidNames_Unix()
         {
             AssertExtensions.Throws<ArgumentException>("name", null, () => new Mutex(new string('a', 1000), options: default));
+            AssertExtensions.Throws<IOException>("name", null, () => new Mutex("Foo/Bar", options: default));
+            AssertExtensions.Throws<IOException>("name", null, () => new Mutex("Foo\\Bar", options: default));
+            AssertExtensions.Throws<IOException>("name", null, () => new Mutex("Global\\Foo/Bar", options: default));
+            AssertExtensions.Throws<IOException>("name", null, () => new Mutex("Global\\Foo\\Bar", options: default));
         }
 
         [Theory]
@@ -971,9 +975,75 @@ public void NamedMutex_DisposeWhenLockedRaceTest()
             }
         }
 
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        public void NamedMutex_OtherEvent_NotCompatible()
+        {
+            using Mutex m = new Mutex(Guid.NewGuid().ToString("N"), options: default);
+            using ManualResetEvent mre = new(false);
+
+            Assert.Throws<PlatformNotSupportedException>(() => WaitHandle.WaitAny(new WaitHandle[] { m, mre }, 0));
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        public void NamedMutex_InvalidSharedMemoryHeaderVersion()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"/tmp/.dotnet/shm/global/{name}";
+            using (FileStream fs = File.Create(path))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)1); // Write the shared memory type (mutex)
+                bw.Write((byte)2); // Write an invalid version number
+                // Make the file large enough for a valid named mutex file and divisible by page size (it should always be under one page).
+                fs.SetLength(Interop.Sys.GetPageSize());
+            }
+
+            Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}"));
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        public void NamedMutex_InvalidSharedMemoryHeaderKind()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"/tmp/.dotnet/shm/global/{name}";
+            using (FileStream fs = File.Create(path))
+            using (BinaryWriter bw = new(fs))
+            {
+                fs.SetLength(Interop.Sys.GetPageSize());
+                fs.Position = 0;
+                bw.Write((byte)2); // Write the shared memory type (invalid)
+                bw.Write((byte)1); // Write a version number
+                // Make the file large enough for a valid named mutex file and divisible by page size (it should always be under one page).
+                fs.SetLength(Interop.Sys.GetPageSize());
+            }
+
+            Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}"));
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        public void NamedMutex_TooSmallSharedMemoryFile()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"/tmp/.dotnet/shm/global/{name}";
+            using (FileStream fs = File.Create(path))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)1); // Write the shared memory type (mutex)
+                bw.Write((byte)1); // Write an valid version number
+                // Make the file large enough for a valid named mutex file but not divisible by page size.
+                fs.SetLength(Interop.Sys.GetPageSize() - 1);
+            }
+
+            Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}"));
+        }
+
         public static TheoryData<string> GetValidNames()
         {
-            var names  =  new TheoryData<string>() { Guid.NewGuid().ToString("N") };
+            var names = new TheoryData<string>() { Guid.NewGuid().ToString("N") };
 
             // Windows native named mutexes and in-proc named mutexes support very long (1000+ char) names.
             // Non-Windows cross-process named mutexes are emulated using file system. It imposes limit

src/libraries/System.Threading/tests/System.Threading.Tests.csproj

@@ -41,5 +41,7 @@
              Link="CommonTest\System\Diagnostics\Tracing\TestEventListener.cs" />
     <Compile Include="$(CommonTestPath)System\Threading\ThreadTestHelpers.cs"
              Link="CommonTest\System\Threading\ThreadTestHelpers.cs" />
+    <Compile Include="$(CommonPath)Interop\Unix\Interop.GetPageSize.cs"
+             Link="Common\Interop\Unix\Interop.GetPageSize.cs" />
   </ItemGroup>
 </Project>

src/native/libs/System.Native/entrypoints.c

@@ -221,6 +221,7 @@ static const Entry s_sysNative[] =
     DllImportEntry(SystemNative_SchedSetAffinity)
     DllImportEntry(SystemNative_SchedGetAffinity)
     DllImportEntry(SystemNative_GetProcessPath)
+    DllImportEntry(SystemNative_GetPageSize)
     DllImportEntry(SystemNative_GetNonCryptographicallySecureRandomBytes)
     DllImportEntry(SystemNative_GetCryptographicallySecureRandomBytes)
     DllImportEntry(SystemNative_GetUnixRelease)

src/native/libs/System.Native/pal_process.c

@@ -899,3 +899,10 @@ char* SystemNative_GetProcessPath(void)
 {
     return minipal_getexepath();
 }
+
+int32_t SystemNative_GetPageSize(void)
+{
+    // The page size is the size of a memory page, which is the smallest unit of memory that can be
+    // allocated or deallocated. It is typically 4096 bytes on most systems.
+    return getpagesize();
+}

src/native/libs/System.Native/pal_process.h

@@ -244,3 +244,5 @@ PALEXPORT int32_t SystemNative_SchedGetAffinity(int32_t pid, intptr_t* mask);
  * resolving symbolic links. The caller is responsible for releasing the buffer.
  */
 PALEXPORT char* SystemNative_GetProcessPath(void);
+
+PALEXPORT int32_t SystemNative_GetPageSize(void);