Add some tests for shared mutex corner cases (#118710)

Author: jkoritzinsky

src/libraries/System.Threading/tests/MutexTests.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
 using System.Threading.Tasks;
 using Microsoft.DotNet.RemoteExecutor;
 using Xunit;
@@ -250,6 +251,11 @@ public void MutualExclusionTest()
         public void Ctor_InvalidNames_Unix()
         {
             AssertExtensions.Throws<ArgumentException>("name", null, () => new Mutex(new string('a', 1000), options: default));
+            Assert.Throws<IOException>(() => new Mutex("Foo/Bar", options: default));
+            AssertExtensions.Throws<ArgumentException>("name", null, () => new Mutex("Foo\\Bar", options: default));
+            AssertExtensions.Throws<ArgumentException>("name", null, () => new Mutex("Foo\\Bar", options: new NamedWaitHandleOptions { CurrentSessionOnly = false }));
+            Assert.Throws<IOException>(() => new Mutex("Global\\Foo/Bar", options: new NamedWaitHandleOptions { CurrentSessionOnly = false }));
+            Assert.Throws<IOException>(() => new Mutex("Global\\Foo\\Bar", options: new NamedWaitHandleOptions { CurrentSessionOnly = false }));
         }
 
         [Theory]
@@ -971,9 +977,118 @@ public void NamedMutex_DisposeWhenLockedRaceTest()
             }
         }
 
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        public void NamedMutex_OtherEvent_NotCompatible()
+        {
+            using Mutex m = new Mutex(Guid.NewGuid().ToString("N"), options: default);
+            using ManualResetEvent mre = new(false);
+
+            Assert.Throws<PlatformNotSupportedException>(() => WaitHandle.WaitAny(new WaitHandle[] { m, mre }, 0));
+        }
+
+        private const string GlobalSharedMemoryDirectory = $"/tmp/.dotnet/shm/global";
+        private const UnixFileMode AllUsersRwx =
+            UnixFileMode.UserRead
+            | UnixFileMode.UserWrite
+            | UnixFileMode.UserExecute
+            | UnixFileMode.GroupRead
+            | UnixFileMode.GroupWrite
+            | UnixFileMode.GroupExecute
+            | UnixFileMode.OtherRead
+            | UnixFileMode.OtherWrite
+            | UnixFileMode.OtherExecute;
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        [UnsupportedOSPlatform("windows")]
+        public void NamedMutex_InvalidSharedMemoryHeaderVersion()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"{GlobalSharedMemoryDirectory}/{name}";
+
+            Directory.CreateDirectory(GlobalSharedMemoryDirectory, AllUsersRwx);
+            using (FileStream fs = new(path, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)1); // Write the shared memory type (mutex)
+                bw.Write((byte)2); // Write an invalid version number
+                // Make the file large enough for a valid named mutex file and divisible by page size (it should always be under one page).
+                fs.SetLength(Environment.SystemPageSize);
+
+                // Try opening a mutex when we still have the file locked.
+                Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}", new NamedWaitHandleOptions { CurrentSessionOnly = false, CurrentUserOnly = false }));
+            }
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        [UnsupportedOSPlatform("windows")]
+        public void NamedMutex_SharedMemoryFileAlreadyOpen()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"{GlobalSharedMemoryDirectory}/{name}";
+
+            Directory.CreateDirectory(GlobalSharedMemoryDirectory, AllUsersRwx);
+            // Take an exclusive file lock of the global shared memory file.
+            using (FileStream fs = new(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)1); // Write the shared memory type (mutex)
+                bw.Write((byte)1); // Write  valid version number
+                // Make the file large enough for a valid named mutex file and divisible by page size (it should always be under one page).
+                fs.SetLength(Environment.SystemPageSize);
+
+                // Try opening a mutex when we still have the file locked.
+                Assert.Throws<IOException>(() => new Mutex($"Global\\{name}", new NamedWaitHandleOptions { CurrentSessionOnly = false, CurrentUserOnly = false }));
+            }
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        [UnsupportedOSPlatform("windows")]
+        public void NamedMutex_InvalidSharedMemoryHeaderKind()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"{GlobalSharedMemoryDirectory}/{name}";
+
+            Directory.CreateDirectory(GlobalSharedMemoryDirectory, AllUsersRwx);
+            using (FileStream fs = new(path, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)2); // Write the shared memory type (invalid)
+                bw.Write((byte)1); // Write a version number
+                // Make the file large enough for a valid named mutex file and divisible by page size (it should always be under one page).
+                fs.SetLength(Environment.SystemPageSize);
+                // Try opening a mutex when we still have the file locked.
+                Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}", new NamedWaitHandleOptions { CurrentSessionOnly = false, CurrentUserOnly = false }));
+            }
+        }
+
+        [ConditionalFact(nameof(IsCrossProcessNamedMutexSupported))]
+        [PlatformSpecific(TestPlatforms.AnyUnix)]
+        [UnsupportedOSPlatform("windows")]
+        public void NamedMutex_TooSmallSharedMemoryFile()
+        {
+            string name = Guid.NewGuid().ToString("N");
+            string path = $"{GlobalSharedMemoryDirectory}/{name}";
+
+            Directory.CreateDirectory(GlobalSharedMemoryDirectory, AllUsersRwx);
+            using (FileStream fs = new(path, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096))
+            using (BinaryWriter bw = new(fs))
+            {
+                bw.Write((byte)1); // Write the shared memory type (mutex)
+                bw.Write((byte)1); // Write a valid version number
+                // Make the file large enough for a valid named mutex file but not divisible by page size.
+                fs.SetLength(Environment.SystemPageSize - 1);
+                // Try opening a mutex when we still have the file locked.
+                Assert.Throws<WaitHandleCannotBeOpenedException>(() => new Mutex($"Global\\{name}", new NamedWaitHandleOptions { CurrentSessionOnly = false, CurrentUserOnly = false }));
+            }
+        }
+
         public static TheoryData<string> GetValidNames()
         {
-            var names  =  new TheoryData<string>() { Guid.NewGuid().ToString("N") };
+            var names = new TheoryData<string>() { Guid.NewGuid().ToString("N") };
 
             // Windows native named mutexes and in-proc named mutexes support very long (1000+ char) names.
             // Non-Windows cross-process named mutexes are emulated using file system. It imposes limit