diff --git a/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj b/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj
index be9203b581a..d84158d112e 100644
--- a/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj
+++ b/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj
@@ -74,4 +74,10 @@
     <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
   </ItemGroup>
 
+  <Target Name="CreateTaskDir" AfterTargets="Build" Condition="'$(TargetFrameworkIdentifier)' != ''">
+    <ItemGroup>
+      <OutputAssemblyList Include="$(TargetDir)Microsoft.Build.CommandLine.UnitTests.dll" />
+    </ItemGroup>
+    <Copy SourceFiles="@(OutputAssemblyList)" DestinationFolder="$(TargetDir)Task" />
+  </Target>
 </Project>
diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs
index 630890e9085..b3c65a114a6 100644
--- a/src/MSBuild.UnitTests/XMake_Tests.cs
+++ b/src/MSBuild.UnitTests/XMake_Tests.cs
@@ -2237,7 +2237,8 @@ public void EndToEndWarnAsErrors()
 
 #if FEATURE_ASSEMBLYLOADCONTEXT
         /// <summary>
-        /// Ensure that tasks get loaded into their own <see cref="System.Runtime.Loader.AssemblyLoadContext"/>.
+        /// Ensure that tasks get loaded into their own <see cref="System.Runtime.Loader.AssemblyLoadContext"/>
+        /// if they are in a directory other than the MSBuild directory.
         /// </summary>
         /// <remarks>
         /// When loading a task from a test assembly in a test within that assembly, the assembly is already loaded
@@ -2247,7 +2248,10 @@ public void EndToEndWarnAsErrors()
         [Fact]
         public void TasksGetAssemblyLoadContexts()
         {
-            string customTaskPath = Assembly.GetExecutingAssembly().Location;
+            string customTaskPath = Path.Combine(
+                Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
+                "Task",
+                Path.GetFileName(Assembly.GetExecutingAssembly().Location));
 
             string projectContents = $@"<Project ToolsVersion=`msbuilddefaulttoolsversion` xmlns=`msbuildnamespace`>
   <UsingTask TaskName=`ValidateAssemblyLoadContext` AssemblyFile=`{customTaskPath}` />
@@ -2259,7 +2263,6 @@ public void TasksGetAssemblyLoadContexts()
 
             ExecuteMSBuildExeExpectSuccess(projectContents);
         }
-
 #endif
 
         private string CopyMSBuild()
diff --git a/src/Shared/CoreCLRAssemblyLoader.cs b/src/Shared/CoreCLRAssemblyLoader.cs
index 14cd04a244d..385c5c878e8 100644
--- a/src/Shared/CoreCLRAssemblyLoader.cs
+++ b/src/Shared/CoreCLRAssemblyLoader.cs
@@ -23,8 +23,15 @@ internal sealed class CoreClrAssemblyLoader
 
         private bool _resolvingHandlerHookedUp = false;
 
+        private static readonly string _msbuildDirPath;
         private static readonly Version _currentAssemblyVersion = new Version(Microsoft.Build.Shared.MSBuildConstants.CurrentAssemblyVersion);
 
+        static CoreClrAssemblyLoader()
+        {
+            _msbuildDirPath = FileUtilities.NormalizePath(typeof(CoreClrAssemblyLoader).Assembly.Location);
+            _msbuildDirPath = Path.GetDirectoryName(_msbuildDirPath);
+        }
+
         public void AddDependencyLocation(string fullPath)
         {
             if (fullPath == null)
@@ -52,7 +59,12 @@ public Assembly LoadFromPath(string fullPath)
             // folders in a NuGet package).
             fullPath = FileUtilities.NormalizePath(fullPath);
 
-            if (Traits.Instance.EscapeHatches.UseSingleLoadContext)
+            // If the requested load comes from the same directory as MSBuild, assume that
+            // the load is part of the platform, and load it using the Default ALC.
+            string assemblyDir = Path.GetDirectoryName(fullPath);
+
+            if (Traits.Instance.EscapeHatches.UseSingleLoadContext ||
+                FileUtilities.ComparePathsNoThrow(assemblyDir, _msbuildDirPath, string.Empty))
             {
                 return LoadUsingLegacyDefaultContext(fullPath);
             }
diff --git a/src/Shared/MSBuildLoadContext.cs b/src/Shared/MSBuildLoadContext.cs
index d44b57480d4..f080c2e05a9 100644
--- a/src/Shared/MSBuildLoadContext.cs
+++ b/src/Shared/MSBuildLoadContext.cs
@@ -25,20 +25,8 @@ internal class MSBuildLoadContext : AssemblyLoadContext
                 "MSBuild",
                 "Microsoft.Build",
                 "Microsoft.Build.Framework",
-                "Microsoft.Build.NuGetSdkResolver",
                 "Microsoft.Build.Tasks.Core",
                 "Microsoft.Build.Utilities.Core",
-                "NuGet.Build.Tasks",
-                "NuGet.Common",
-                "NuGet.Configuration",
-                "NuGet.Credentials",
-                "NuGet.DependencyResolver.Core",
-                "NuGet.Frameworks",
-                "NuGet.LibraryModel",
-                "NuGet.Packaging",
-                "NuGet.Protocol",
-                "NuGet.ProjectModel",
-                "NuGet.Versioning",
             }.ToImmutableHashSet();
 
         internal static readonly string[] Extensions = new[] { "ni.dll", "ni.exe", "dll", "exe" };