Fix FrameworkDependentAppLaunch.AppHost_GlobalLocation 'Failed to restore file' CI failure (#116831)

We're hitting an intermittent failure (particularly on Debug) trying to copy over files immediately after the process exits. Since this passes the majority of the time, it does not seem to indicate an actual issue where the files are left in use.
- Increase the existing retry we have for doing this restore
- Move tests to InstallLocation test class as it fits better there (testing registered and default install locations)
- Make InstallLocation test class create one app with test behaviour enabled instead of repeatedly doing it in half of its tests cases.

This also removes explicitly running the app with the app directory as the working directory, which doesn't seem interesting for us to keep testing.

Author: elinor-fung

src/installer/tests/HostActivation.Tests/FrameworkDependentAppLaunch.cs

@@ -187,83 +187,17 @@ public void AppHost()
             if (Binaries.CetCompat.IsSupported)
                 Assert.True(Binaries.CetCompat.IsMarkedCompatible(appExe));
 
-            // Get the framework location that was built
-            string builtDotnet = TestContext.BuiltDotNet.BinPath;
-
-            // Verify running with the default working directory
             Command.Create(appExe)
                 .CaptureStdErr()
                 .CaptureStdOut()
-                .DotNetRoot(builtDotnet, TestContext.BuildArchitecture)
-                .MultilevelLookup(false)
-                .Execute()
-                .Should().Pass()
-                .And.HaveStdOutContaining("Hello World")
-                .And.HaveStdOutContaining(TestContext.MicrosoftNETCoreAppVersion);
-
-
-            // Verify running from within the working directory
-            Command.Create(appExe)
-                .WorkingDirectory(sharedTestState.App.Location)
-                .DotNetRoot(builtDotnet, TestContext.BuildArchitecture)
+                .DotNetRoot(TestContext.BuiltDotNet.BinPath, TestContext.BuildArchitecture)
                 .MultilevelLookup(false)
-                .CaptureStdErr()
-                .CaptureStdOut()
                 .Execute()
                 .Should().Pass()
                 .And.HaveStdOutContaining("Hello World")
                 .And.HaveStdOutContaining(TestContext.MicrosoftNETCoreAppVersion);
         }
 
-        [Theory]
-        [InlineData(true)]
-        [InlineData(false)]
-        public void AppHost_GlobalLocation(bool useRegisteredLocation)
-        {
-            string appExe = sharedTestState.App.AppExe;
-
-            // Get the framework location that was built
-            string builtDotnet = TestContext.BuiltDotNet.BinPath;
-
-            using (var registeredInstallLocationOverride = new RegisteredInstallLocationOverride(appExe))
-            {
-                string architecture = TestContext.BuildArchitecture;
-                if (useRegisteredLocation)
-                {
-                    registeredInstallLocationOverride.SetInstallLocation(new (string, string)[] { (architecture, builtDotnet) });
-                }
-
-                // Verify running with the default working directory
-                Command.Create(appExe)
-                    .CaptureStdErr()
-                    .CaptureStdOut()
-                    .MultilevelLookup(false)
-                    .ApplyRegisteredInstallLocationOverride(registeredInstallLocationOverride)
-                    .EnvironmentVariable(Constants.TestOnlyEnvironmentVariables.DefaultInstallPath, useRegisteredLocation ? null : builtDotnet)
-                    .DotNetRoot(null)
-                    .Execute()
-                    .Should().Pass()
-                    .And.HaveStdOutContaining("Hello World")
-                    .And.HaveStdOutContaining(TestContext.MicrosoftNETCoreAppVersion)
-                    .And.NotHaveStdErr();
-
-                // Verify running from within the working directory
-                Command.Create(appExe)
-                    .CaptureStdErr()
-                    .CaptureStdOut()
-                    .MultilevelLookup(false)
-                    .WorkingDirectory(sharedTestState.App.Location)
-                    .ApplyRegisteredInstallLocationOverride(registeredInstallLocationOverride)
-                    .EnvironmentVariable(Constants.TestOnlyEnvironmentVariables.DefaultInstallPath, useRegisteredLocation ? null : builtDotnet)
-                    .DotNetRoot(null)
-                    .Execute()
-                    .Should().Pass()
-                    .And.HaveStdOutContaining("Hello World")
-                    .And.HaveStdOutContaining(TestContext.MicrosoftNETCoreAppVersion)
-                    .And.NotHaveStdErr();
-            }
-        }
-
         [ConditionalFact(typeof(Binaries.CetCompat), nameof(Binaries.CetCompat.IsSupported))]
         public void AppHost_DisableCetCompat()
         {

src/installer/tests/HostActivation.Tests/InstallLocation.cs

@@ -64,7 +64,7 @@ public void EnvironmentVariable_ArchSpecificDotnetRootIsUsedOverDotnetRoot()
         [Fact]
         public void EnvironmentVariable_DotNetRootIsUsedOverInstallLocationIfSet()
         {
-            var app = sharedTestState.App.Copy();
+            var app = sharedTestState.TestBehaviourEnabledApp;
             var appExe = app.AppExe;
             var arch = TestContext.BuildArchitecture.ToUpper();
             var dotnet = TestContext.BuiltDotNet.BinPath;
@@ -87,44 +87,38 @@ public void EnvironmentVariable_DotNetRootIsUsedOverInstallLocationIfSet()
         [Fact]
         public void EnvironmentVariable_DotnetRootPathDoesNotExist()
         {
-            var app = sharedTestState.App.Copy();
-            using (TestOnlyProductBehavior.Enable(app.AppExe))
-            {
-                Command.Create(app.AppExe)
-                    .EnableTracingAndCaptureOutputs()
-                    .DotNetRoot("non_existent_path")
-                    .MultilevelLookup(false)
-                    .EnvironmentVariable(
-                        Constants.TestOnlyEnvironmentVariables.GloballyRegisteredPath,
-                        TestContext.BuiltDotNet.BinPath)
-                    .Execute()
-                    .Should().Pass()
-                    .And.HaveStdErrContaining("Did not find [DOTNET_ROOT] directory [non_existent_path]")
-                    // If DOTNET_ROOT points to a folder that does not exist, we fall back to the global install path.
-                    .And.HaveUsedGlobalInstallLocation(TestContext.BuiltDotNet.BinPath)
-                    .And.HaveStdOutContaining("Hello World");
-            }
+            var app = sharedTestState.TestBehaviourEnabledApp;
+            Command.Create(app.AppExe)
+                .EnableTracingAndCaptureOutputs()
+                .DotNetRoot("non_existent_path")
+                .MultilevelLookup(false)
+                .EnvironmentVariable(
+                    Constants.TestOnlyEnvironmentVariables.GloballyRegisteredPath,
+                    TestContext.BuiltDotNet.BinPath)
+                .Execute()
+                .Should().Pass()
+                .And.HaveStdErrContaining("Did not find [DOTNET_ROOT] directory [non_existent_path]")
+                // If DOTNET_ROOT points to a folder that does not exist, we fall back to the global install path.
+                .And.HaveUsedGlobalInstallLocation(TestContext.BuiltDotNet.BinPath)
+                .And.HaveStdOutContaining("Hello World");
         }
 
         [Fact]
         public void EnvironmentVariable_DotnetRootPathExistsButHasNoHost()
         {
-            var app = sharedTestState.App.Copy();
-            using (TestOnlyProductBehavior.Enable(app.AppExe))
-            {
-                Command.Create(app.AppExe)
-                    .EnableTracingAndCaptureOutputs()
-                    .DotNetRoot(app.Location)
-                    .MultilevelLookup(false)
-                    .EnvironmentVariable(
-                        Constants.TestOnlyEnvironmentVariables.GloballyRegisteredPath,
-                        TestContext.BuiltDotNet.BinPath)
-                    .Execute()
-                    .Should().Fail()
-                    .And.HaveUsedDotNetRootInstallLocation(app.Location, TestContext.BuildRID)
-                    // If DOTNET_ROOT points to a folder that exists we assume that there's a dotnet installation in it
-                    .And.HaveStdErrContaining($"The required library {Binaries.HostFxr.FileName} could not be found.");
-            }
+            var app = sharedTestState.TestBehaviourEnabledApp;
+            Command.Create(app.AppExe)
+                .EnableTracingAndCaptureOutputs()
+                .DotNetRoot(app.Location)
+                .MultilevelLookup(false)
+                .EnvironmentVariable(
+                    Constants.TestOnlyEnvironmentVariables.GloballyRegisteredPath,
+                    TestContext.BuiltDotNet.BinPath)
+                .Execute()
+                .Should().Fail()
+                .And.HaveUsedDotNetRootInstallLocation(app.Location, TestContext.BuildRID)
+                // If DOTNET_ROOT points to a folder that exists we assume that there's a dotnet installation in it
+                .And.HaveStdErrContaining($"The required library {Binaries.HostFxr.FileName} could not be found.");
         }
 
         [Fact]
@@ -163,10 +157,49 @@ public void EnvironmentVariable_DotNetInfo_ListEnvironment()
             }
         }
 
+        [Fact]
+        public void DefaultInstallLocation()
+        {
+            TestApp app = sharedTestState.TestBehaviourEnabledApp;
+
+            // Ensure no install locations are registered, so the default install location is used
+            using (var registeredInstallLocationOverride = new RegisteredInstallLocationOverride(app.AppExe))
+            {
+                Command.Create(app.AppExe)
+                    .EnableTracingAndCaptureOutputs()
+                    .ApplyRegisteredInstallLocationOverride(registeredInstallLocationOverride)
+                    .EnvironmentVariable(Constants.TestOnlyEnvironmentVariables.DefaultInstallPath, TestContext.BuiltDotNet.BinPath)
+                    .DotNetRoot(null)
+                    .Execute()
+                    .Should().Pass()
+                    .And.HaveUsedGlobalInstallLocation(TestContext.BuiltDotNet.BinPath);
+            }
+        }
+
+        [Fact]
+        public void RegisteredInstallLocation()
+        {
+            TestApp app = sharedTestState.TestBehaviourEnabledApp;
+            using (var registeredInstallLocationOverride = new RegisteredInstallLocationOverride(app.AppExe))
+            {
+                registeredInstallLocationOverride.SetInstallLocation(
+                    (TestContext.BuildArchitecture, TestContext.BuiltDotNet.BinPath));
+
+                Command.Create(app.AppExe)
+                    .EnableTracingAndCaptureOutputs()
+                    .ApplyRegisteredInstallLocationOverride(registeredInstallLocationOverride)
+                    .DotNetRoot(null)
+                    .Execute()
+                    .Should().Pass()
+                    .And.HaveUsedRegisteredInstallLocation(TestContext.BuiltDotNet.BinPath)
+                    .And.HaveUsedGlobalInstallLocation(TestContext.BuiltDotNet.BinPath);
+            }
+        }
+
         [Fact]
         public void RegisteredInstallLocation_ArchSpecificLocationIsPickedFirst()
         {
-            var app = sharedTestState.App.Copy();
+            var app = sharedTestState.TestBehaviourEnabledApp;
             var arch1 = "someArch";
             var path1 = "x/y/z";
             var arch2 = TestContext.BuildArchitecture;
@@ -202,7 +235,7 @@ public void RegisteredInstallLocation_ArchSpecificLocationIsPickedFirst()
         [SkipOnPlatform(TestPlatforms.Windows, "This test targets the install_location config file which is only used on Linux and macOS.")]
         public void InstallLocationFile_ReallyLongInstallPathIsParsedCorrectly()
         {
-            var app = sharedTestState.App.Copy();
+            var app = sharedTestState.TestBehaviourEnabledApp;
             using (var registeredInstallLocationOverride = new RegisteredInstallLocationOverride(app.AppExe))
             {
                 var reallyLongPath =
@@ -225,9 +258,8 @@ public void InstallLocationFile_ReallyLongInstallPathIsParsedCorrectly()
         [SkipOnPlatform(TestPlatforms.Windows, "This test targets the install_location config file which is only used on Linux and macOS.")]
         public void InstallLocationFile_MissingFile()
         {
-            var app = sharedTestState.App.Copy();
+            var app = sharedTestState.TestBehaviourEnabledApp;
             using (var testArtifact = TestArtifact.Create("missingInstallLocation"))
-            using (var testOnlyProductBehavior = TestOnlyProductBehavior.Enable(app.AppExe))
             {
                 string installLocationDirectory = Path.Combine(testArtifact.Location, "installLocationOverride");
                 Directory.CreateDirectory(installLocationDirectory);
@@ -345,7 +377,7 @@ public void SearchOptions(SearchLocation searchLocation)
         }
 
         [Fact]
-        public void AppHost_AppRelative_MissingPath()
+        public void SearchOptions_AppRelative_MissingPath()
         {
             TestApp app = sharedTestState.App.Copy();
             app.CreateAppHost(dotNetRootOptions: new HostWriter.DotNetSearchOptions()
@@ -450,16 +482,25 @@ public void SearchOptions_Precedence(SearchLocation expectedResult)
         public class SharedTestState : IDisposable
         {
             public TestApp App { get; }
+            public TestApp TestBehaviourEnabledApp { get; }
 
             public SharedTestState()
             {
                 App = TestApp.CreateFromBuiltAssets("HelloWorld");
                 App.CreateAppHost();
+
+                TestBehaviourEnabledApp = TestApp.CreateFromBuiltAssets("HelloWorld");
+                TestBehaviourEnabledApp.CreateAppHost();
+
+                // Enable test-only behavior for the app. We don't bother disabling the behaviour later,
+                // as we just delete the entire app after the tests run.
+                _ = TestOnlyProductBehavior.Enable(TestBehaviourEnabledApp.AppExe);
             }
 
             public void Dispose()
             {
                 App?.Dispose();
+                TestBehaviourEnabledApp?.Dispose();
             }
         }
     }

src/installer/tests/TestUtils/TestFileBackup.cs

@@ -116,7 +116,7 @@ private static void CopyOverDirectory(string source, string destination)
             }
         }
 
-        private static void RetryOnIOError(Func<bool> action, string errorMessage, int maxRetries = 5)
+        private static void RetryOnIOError(Func<bool> action, string errorMessage, int maxRetries = 25)
         {
             IOException exception = null;
             for (int i = 0; i < maxRetries; i++)