diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs
index 3032b80f93a..7a70791f17b 100644
--- a/src/Build/BackEnd/BuildManager/BuildManager.cs
+++ b/src/Build/BackEnd/BuildManager/BuildManager.cs
@@ -1151,11 +1151,6 @@ public void EndBuild()
                     Reset();
                     _buildManagerState = BuildManagerState.Idle;
 
-                    if (Traits.Instance.ForceTaskFactoryOutOfProc || _buildParameters.MultiThreaded)
-                    {
-                        TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
-                    }
-
                     MSBuildEventSource.Log.BuildStop();
 
                     _threadException?.Throw();
diff --git a/src/Shared/TaskFactoryUtilities.cs b/src/Shared/TaskFactoryUtilities.cs
index 0f671430f87..1dc8353799e 100644
--- a/src/Shared/TaskFactoryUtilities.cs
+++ b/src/Shared/TaskFactoryUtilities.cs
@@ -27,10 +27,6 @@ namespace Microsoft.Build.Shared
     /// </summary>
     internal static class TaskFactoryUtilities
     {
-        /// <summary>
-        /// The sub-path within the temporary directory where compiled inline tasks are located.
-        /// </summary>
-        public const string InlineTaskTempDllSubPath = nameof(InlineTaskTempDllSubPath);
         public const string InlineTaskSuffix = "inline_task.dll";
         public const string InlineTaskLoadManifestSuffix = ".loadmanifest";
 
@@ -57,30 +53,13 @@ public CachedAssemblyEntry(Assembly assembly, string assemblyPath)
             public bool IsValid => string.IsNullOrEmpty(AssemblyPath) || FileUtilities.FileExistsNoThrow(AssemblyPath);
         }
 
-
-        /// <summary>
-        /// Creates a process-specific temporary directory for inline task assemblies.
-        /// </summary>
-        /// <returns>The path to the created temporary directory.</returns>
-        public static string CreateProcessSpecificTemporaryTaskDirectory()
-        {
-            string processSpecificInlineTaskDir = Path.Combine(
-                FileUtilities.TempFileDirectory,
-                InlineTaskTempDllSubPath,
-                $"pid_{EnvironmentUtilities.CurrentProcessId}");
-
-            Directory.CreateDirectory(processSpecificInlineTaskDir);
-            return processSpecificInlineTaskDir;
-        }
-
         /// <summary>
         /// Gets a temporary file path for an inline task assembly in the process-specific directory.
         /// </summary>
         /// <returns>The full path to the temporary file.</returns>
         public static string GetTemporaryTaskAssemblyPath()
         {
-            string taskDir = CreateProcessSpecificTemporaryTaskDirectory();
-            return FileUtilities.GetTemporaryFile(taskDir, fileName: null, extension: "inline_task.dll", createFile: false);
+            return FileUtilities.GetTemporaryFile(directory: null, fileName: null, extension: "inline_task.dll", createFile: false);
         }
 
         /// <summary>
@@ -256,28 +235,6 @@ public static bool ShouldCompileForOutOfProcess(IBuildEngine taskFactoryEngineCo
             return false;
         }
 
-        /// <summary>
-        /// Cleans up the current process's inline task directory by deleting the temporary directory
-        /// and its contents used for inline task assemblies for this specific process.
-        /// This should be called at the end of a build to prevent dangling DLL files.
-        /// </summary>
-        /// <remarks>
-        /// On Windows platforms, this may fail to delete files that are still locked by the current process.
-        /// However, it will clean up any files that are no longer in use.
-        /// </remarks>
-        public static void CleanCurrentProcessInlineTaskDirectory()
-        {
-            string processSpecificInlineTaskDir = Path.Combine(
-                FileUtilities.TempFileDirectory,
-                InlineTaskTempDllSubPath,
-                $"pid_{EnvironmentUtilities.CurrentProcessId}");
-                
-            if (FileSystems.Default.DirectoryExists(processSpecificInlineTaskDir))
-            {
-                FileUtilities.DeleteDirectoryNoThrow(processSpecificInlineTaskDir, recursive: true);
-            }
-        }
-
         /// <summary>
         /// Attempts to load an assembly by searching in the specified directories.
         /// </summary>
diff --git a/src/Shared/TempFileUtilities.cs b/src/Shared/TempFileUtilities.cs
index 2b77811c660..909b9863dc1 100644
--- a/src/Shared/TempFileUtilities.cs
+++ b/src/Shared/TempFileUtilities.cs
@@ -4,6 +4,7 @@
 using System;
 using System.IO;
 using System.Runtime.CompilerServices;
+using System.Threading;
 using Microsoft.Build.Shared.FileSystem;
 
 #nullable disable
@@ -16,40 +17,58 @@ namespace Microsoft.Build.Shared
     /// </summary>
     internal static partial class FileUtilities
     {
-        // For the current user, these correspond to read, write, and execute permissions.
-        // Lower order bits correspond to the same for "group" or "other" users.
-        private static string tempFileDirectory = null;
+        private static Lazy<string> tempFileDirectory = CreateTempFileDirectoryLazy();
+
         private const string msbuildTempFolderPrefix = "MSBuildTemp";
 
-        internal static string TempFileDirectory
+        internal static string TempFileDirectory => tempFileDirectory.Value;
+
+        private static Lazy<string> CreateTempFileDirectoryLazy()
+        {
+            return new Lazy<string>(
+                () =>
+                {
+                    string path = CreateFolderUnderTemp();
+                    RegisterCleanupOnExit(path);
+                    return path;
+                },
+                LazyThreadSafetyMode.ExecutionAndPublication);
+        }
+
+        private static void RegisterCleanupOnExit(string pathToCleanup)
         {
-            get
+            AppDomain.CurrentDomain.ProcessExit += (_, _) =>
             {
-                return tempFileDirectory ??= CreateFolderUnderTemp();
-            }
+                try
+                {
+                    if (Directory.Exists(pathToCleanup))
+                    {
+                        Directory.Delete(pathToCleanup, recursive: true);
+                    }
+                }
+                catch
+                {
+                    // Best effort - ignore failures during cleanup
+                }
+            };
         }
 
         internal static void ClearTempFileDirectory()
         {
-            tempFileDirectory = null;
+            tempFileDirectory = CreateTempFileDirectoryLazy();
         }
 
-        // For all native calls, directly check their return values to prevent bad actors from getting in between checking if a directory exists and returning it.
         private static string CreateFolderUnderTemp()
         {
-            string path = null;
-
-            if (NativeMethodsShared.IsLinux)
-            {
-#if NET // always true, Linux implies NET
-                path = Directory.CreateTempSubdirectory(msbuildTempFolderPrefix).FullName;
+            string path;
+
+#if NET
+            path = Directory.CreateTempSubdirectory(msbuildTempFolderPrefix).FullName;
+#else
+            // CreateTempSubdirectory API is not available in .NET Framework
+            path = Path.Combine(Path.GetTempPath(), $"{msbuildTempFolderPrefix}{Guid.NewGuid():N}");
+            Directory.CreateDirectory(path);
 #endif
-            }
-            else
-            {
-                path = Path.Combine(Path.GetTempPath(), msbuildTempFolderPrefix);
-                Directory.CreateDirectory(path);
-            }
 
             return FileUtilities.EnsureTrailingSlash(path);
         }
diff --git a/src/Tasks.UnitTests/CodeTaskFactoryTests.cs b/src/Tasks.UnitTests/CodeTaskFactoryTests.cs
index b61dd893659..2ab31ff10f8 100644
--- a/src/Tasks.UnitTests/CodeTaskFactoryTests.cs
+++ b/src/Tasks.UnitTests/CodeTaskFactoryTests.cs
@@ -758,8 +758,6 @@ public void BuildTaskSimpleCodeFactoryTestExtraReference(bool forceOutOfProc)
         [Fact]
         public void OutOfProcCodeTaskFactoryCachesAssemblyPath()
         {
-            TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
-
             try
                         {
                                 const string taskElementContents = @"<Code Type=""Fragment"" Language=""cs"">
@@ -805,7 +803,6 @@ public void OutOfProcCodeTaskFactoryCachesAssemblyPath()
             }
             finally
             {
-                TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
             }
         }
 
diff --git a/src/Tasks.UnitTests/RoslynCodeTaskFactory_Tests.cs b/src/Tasks.UnitTests/RoslynCodeTaskFactory_Tests.cs
index 4992ba064f7..814ea1e440a 100644
--- a/src/Tasks.UnitTests/RoslynCodeTaskFactory_Tests.cs
+++ b/src/Tasks.UnitTests/RoslynCodeTaskFactory_Tests.cs
@@ -238,8 +238,6 @@ public void RoslynCodeTaskFactory_ReuseCompilation(bool forceOutOfProc)
         [Fact]
         public void OutOfProcRoslynTaskFactoryCachesAssemblyPath()
         {
-            TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
-
             try
             {
                 const string taskBody = @"
@@ -285,7 +283,6 @@ public void OutOfProcRoslynTaskFactoryCachesAssemblyPath()
             }
             finally
             {
-                TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
             }
         }
 
diff --git a/src/Tasks.UnitTests/TaskFactoryUtilities_Tests.cs b/src/Tasks.UnitTests/TaskFactoryUtilities_Tests.cs
index 1bc4386371e..4e10d18e565 100644
--- a/src/Tasks.UnitTests/TaskFactoryUtilities_Tests.cs
+++ b/src/Tasks.UnitTests/TaskFactoryUtilities_Tests.cs
@@ -26,7 +26,6 @@ public void GetTemporaryTaskAssemblyPath_ShouldReturnValidPath()
             // Assert
             assemblyPath.ShouldNotBeNull();
             assemblyPath.ShouldEndWith(".dll");
-            Path.GetDirectoryName(assemblyPath).ShouldContain(TaskFactoryUtilities.InlineTaskTempDllSubPath);
         }
 
         [Fact]
diff --git a/src/Tasks.UnitTests/XamlTaskFactory_Tests.cs b/src/Tasks.UnitTests/XamlTaskFactory_Tests.cs
index 370d6dd587b..8eb7e642931 100644
--- a/src/Tasks.UnitTests/XamlTaskFactory_Tests.cs
+++ b/src/Tasks.UnitTests/XamlTaskFactory_Tests.cs
@@ -450,8 +450,6 @@ public class CompilationTests
     [Fact]
     public void OutOfProcXamlTaskFactoryProvidesAssemblyPath()
     {
-      TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
-
       try
       {
         const string taskElementContents = @"<ProjectSchemaDefinitions xmlns=""clr-namespace:Microsoft.Build.Framework.XamlTypes;assembly=Microsoft.Build.Framework"" xmlns:x=""http://schemas.microsoft.com/winfx/2006/xaml"">
@@ -480,7 +478,6 @@ public void OutOfProcXamlTaskFactoryProvidesAssemblyPath()
       }
       finally
       {
-        TaskFactoryUtilities.CleanCurrentProcessInlineTaskDirectory();
       }
     }
 
diff --git a/src/UnitTests.Shared/TestEnvironment.cs b/src/UnitTests.Shared/TestEnvironment.cs
index 1525ccbfa94..586afed20e0 100644
--- a/src/UnitTests.Shared/TestEnvironment.cs
+++ b/src/UnitTests.Shared/TestEnvironment.cs
@@ -599,7 +599,14 @@ public TransientTempPath(string tempPath, bool deleteTempDirectory)
             TempPath = tempPath;
             _deleteTempDirectory = deleteTempDirectory;
 
+            // Ensure the temp directory exists before setting it as TMPDIR
+            // This is required because Directory.CreateTempSubdirectory() expects TMPDIR to exist
+            Directory.CreateDirectory(tempPath);
+
             _oldtempPaths = SetTempPath(tempPath);
+            
+            // Clear the cached temp directory so FileUtilities picks up the new TMPDIR/TMP/TEMP
+            FileUtilities.ClearTempFileDirectory();
         }
 
         private static TempPaths SetTempPath(string tempPath)
@@ -645,6 +652,9 @@ private static TempPaths GetTempPaths()
         public override void Revert()
         {
             SetTempPaths(_oldtempPaths);
+            
+            // Clear the cached temp directory so FileUtilities picks up the restored TMPDIR/TMP/TEMP
+            FileUtilities.ClearTempFileDirectory();
 
             if (_deleteTempDirectory)
             {