diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index eb38f6d96a3..fc7940289f0 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -20,6 +20,14 @@ jobs: steps: - powershell: | $versionsFile = "eng/Versions.props" + + [xml]$xml = Get-Content $versionsFile + $finalVersionKind = $xml.Project.PropertyGroup.DotNetFinalVersionKind + if ($finalVersionKind -ne 'release') { + Write-Host "Since it is not released, skip the version bump check."; + return + } + $changedFiles = git diff --name-only HEAD HEAD~1 $changedVersionsFile = $changedFiles | Where-Object { $_ -eq $versionsFile } $isInitialCommit = $false diff --git a/documentation/High-level-overview.md b/documentation/High-level-overview.md index 4ee9aaa9e30..6cff44e8454 100644 --- a/documentation/High-level-overview.md +++ b/documentation/High-level-overview.md @@ -149,6 +149,15 @@ TaskHost can be opted-in via `TaskFactory="TaskHostFactory"` in the [`UsingTask` - If a task's source code is in the same repository that is being built, and the repository's build needs to use that task during the build process. Using a Task Host makes sure the DLLs are not locked at the end of the build (as MSBuild uses long living worker nodes that survives single build execution) - As an isolation mechanism - separating the execution from the engine execution process. +When `TaskHostFactory` is specified as the task factory, the task always runs out-of-process and short lived. See the below matrix: + +| Does TaskHost match executing MSBuild Runtime? | Is TaskHostFactory requested for the Task? | Expected task execution type | +| :-: | :-: | --- | +| ✅ | :x: | in-process execution | +| ✅ | ✅ | short-lived out-of-proc execution | +| :x: | ✅ | short-lived out-of-proc execution | +| :x: | :x: | long-lived out-of-proc execution | + ## Caches ### Project result cache The project Result Cache refers to the cache used by the scheduler that keeps the build results of already executed project. The result of a target is success, failure, and a list of items that succeeded. Beyond that, the `Returns` and `Outputs` attributes from targets are also serialized with the build result, as to be used by other targets for their execution. diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 5b13936e49b..a0fcf9f8095 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -24,8 +24,8 @@ This file should be imported by eng/Versions.props 9.0.11 9.0.11 - 10.0.0-beta.25626.5 - 10.0.0-beta.25626.5 + 10.0.0-beta.26062.3 + 10.0.0-beta.26062.3 7.3.0-preview.1.50 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1508f78d1ec..272731ff355 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -106,9 +106,9 @@ - + https://github.com/dotnet/arcade - d8dca0b41b903e7182e64543773390b969dab96b + 9f518f2be968c4c0102c2e3f8c793c5b7f28b731 https://github.com/nuget/nuget.client @@ -118,9 +118,9 @@ https://github.com/dotnet/roslyn df7b5aaff073486376dad5d30b6d0ba45595d97d - + https://github.com/dotnet/arcade - d8dca0b41b903e7182e64543773390b969dab96b + 9f518f2be968c4c0102c2e3f8c793c5b7f28b731 diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 3437087c80f..b955fac6e13 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -80,7 +80,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2022.amd64 os: windows steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 9423d71ca3a..b942a79ef02 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -293,11 +293,11 @@ stages: ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2022.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index e0b19c14a07..18693ea120d 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 +# demands: ImageOverride -equals windows.vs2022.amd64 variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - template: /eng/common/templates-official/variables/pool-providers.yml diff --git a/global.json b/global.json index 677476b7c8a..0d6a2066c27 100644 --- a/global.json +++ b/global.json @@ -14,6 +14,6 @@ "xcopy-msbuild": "18.0.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25626.5" + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26062.3" } } diff --git a/src/Build.UnitTests/TaskHostFactoryLifecycle_E2E_Tests.cs b/src/Build.UnitTests/TaskHostFactoryLifecycle_E2E_Tests.cs new file mode 100644 index 00000000000..c67a5ab1a12 --- /dev/null +++ b/src/Build.UnitTests/TaskHostFactoryLifecycle_E2E_Tests.cs @@ -0,0 +1,120 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; +using Microsoft.Build.UnitTests; +using Microsoft.Build.UnitTests.Shared; +using Shouldly; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Build.Engine.UnitTests +{ + /// + /// End-to-end tests for task host factory lifecycle behavior. + /// + /// Tests validate the behavior based on whether the TaskHost runtime matches + /// the executing MSBuild runtime and whether TaskHostFactory is explicitly requested. + /// + /// This is a regression test for https://github.com/dotnet/msbuild/issues/13013 + /// + public class TaskHostFactoryLifecycle_E2E_Tests + { + private static string AssemblyLocation { get; } = Path.Combine(Path.GetDirectoryName(typeof(TaskHostFactoryLifecycle_E2E_Tests).Assembly.Location) ?? System.AppContext.BaseDirectory); + + private static string TestAssetsRootPath { get; } = Path.Combine(AssemblyLocation, "TestAssets", "TaskHostLifecycle"); + + private readonly ITestOutputHelper _output; + + public TaskHostFactoryLifecycle_E2E_Tests(ITestOutputHelper output) + { + _output = output; + } + + /// + /// Validates task host lifecycle behavior for all scenarios. + /// + /// Test scenarios: + /// 1. Runtime matches + TaskHostFactory requested → short-lived out of proc (nodereuse:False) + /// 2. Runtime matches + TaskHostFactory NOT requested → in-proc execution + /// 3. Runtime doesn't match + TaskHostFactory requested → short-lived out of proc (nodereuse:False) + /// 4. Runtime doesn't match + TaskHostFactory NOT requested → long-lived sidecar out of proc (nodereuse:True) + /// + /// The runtime to use for the task (CurrentRuntime or NET) + /// The task factory to use (TaskHostFactory or AssemblyTaskFactory) + [Theory] +#if NET + [InlineData("CurrentRuntime", "AssemblyTaskFactory")] // Match + No Explicit → in-proc + [InlineData("CurrentRuntime", "TaskHostFactory")] // Match + Explicit → short-lived out-of-proc +#endif + [InlineData("NET", "AssemblyTaskFactory")] // No Match + No Explicit → long-lived sidecar out-of-proc + [InlineData("NET", "TaskHostFactory")] // No Match + Explicit → short-lived out-of-proc + public void TaskHostLifecycle_ValidatesAllScenarios( + string runtimeToUse, + string taskFactoryToUse) + { + bool? expectedNodeReuse; + + // TaskHostFactory is always short lived and out-of-proc + if (taskFactoryToUse == "TaskHostFactory") + { + expectedNodeReuse = false; + } + // AssemblyTaskFactory behavior depends on runtime + else if (taskFactoryToUse == "AssemblyTaskFactory") + { + if (runtimeToUse == "CurrentRuntime") + { + // in-proc + expectedNodeReuse = null; + } + else if (runtimeToUse == "NET") + { + // When running on .NET Framework: out-of-proc, otherwise on .NET in-proc. + expectedNodeReuse = RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase) ? true : null; + } + else + { + throw new ArgumentOutOfRangeException(nameof(runtimeToUse), "Unknown runtime to use: " + runtimeToUse); + } + } + else + { + throw new ArgumentOutOfRangeException(nameof(taskFactoryToUse), "Unknown task factory to use: " + taskFactoryToUse); + } + + using TestEnvironment env = TestEnvironment.Create(_output); + string testProjectPath = Path.Combine(TestAssetsRootPath, "TaskHostLifecycleTestApp.csproj"); + + string testTaskOutput = RunnerUtilities.ExecBootstrapedMSBuild( + $"{testProjectPath} -v:n -restore /p:RuntimeToUse={runtimeToUse} /p:TaskFactoryToUse={taskFactoryToUse}", + out bool successTestTask, + outputHelper: _output); + + successTestTask.ShouldBeTrue(); + + // Verify execution mode (out-of-proc vs in-proc) and node reuse behavior + if (expectedNodeReuse.HasValue) + { + // For out-of-proc scenarios, validate the task runs in a separate process + // by checking for the presence of command-line arguments that indicate task host execution + testTaskOutput.ShouldContain("/nodemode:", + customMessage: "Task should run out-of-proc and have /nodemode: in its command-line arguments"); + + // Validate the nodereuse flag in the task's command-line arguments + string expectedFlag = expectedNodeReuse.Value ? "/nodereuse:True" : "/nodereuse:False"; + testTaskOutput.ShouldContain(expectedFlag, + customMessage: $"Task should have {expectedFlag} in its command-line arguments"); + } + else + { + // For in-proc scenarios, validate the task does NOT run in a task host + // by ensuring task host specific command-line flags are not present + testTaskOutput.ShouldNotContain("/nodemode:", + customMessage: "Task should run in-proc and not have task host command-line arguments like /nodemode:"); + } + } + } +} diff --git a/src/Build.UnitTests/TestAssets/ExampleNetTask/ExampleTask/ExampleTask.cs b/src/Build.UnitTests/TestAssets/ExampleNetTask/ExampleTask/ExampleTask.cs index 563724a07f7..1a321e328cd 100644 --- a/src/Build.UnitTests/TestAssets/ExampleNetTask/ExampleTask/ExampleTask.cs +++ b/src/Build.UnitTests/TestAssets/ExampleNetTask/ExampleTask/ExampleTask.cs @@ -20,7 +20,7 @@ public override bool Execute() var executingProcess = currentProcess.ProcessName; var processPath = currentProcess.MainModule?.FileName ?? "Unknown"; - Log.LogMessage(MessageImportance.High, $"The task is executed in process: {executingProcess}"); + Log.LogMessage(MessageImportance.High, $"The task is executed in process: {executingProcess} with id {currentProcess.Id}"); Log.LogMessage(MessageImportance.High, $"Process path: {processPath}"); string[] args = Environment.GetCommandLineArgs(); diff --git a/src/Build.UnitTests/TestAssets/TaskHostLifecycle/TaskHostLifecycleTestApp.csproj b/src/Build.UnitTests/TestAssets/TaskHostLifecycle/TaskHostLifecycleTestApp.csproj new file mode 100644 index 00000000000..adb43e0de99 --- /dev/null +++ b/src/Build.UnitTests/TestAssets/TaskHostLifecycle/TaskHostLifecycleTestApp.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + $([System.IO.Path]::GetFullPath('$([System.IO.Path]::Combine('$(AssemblyLocation)', '..'))')) + $([System.IO.Path]::Combine('$(TestProjectFolder)', '$(TargetFramework)', 'ExampleTask.dll')) + + + + + + + + + diff --git a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs index e5eecc0ef06..9dabd768dca 100644 --- a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs +++ b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs @@ -216,6 +216,7 @@ protected IList GetNodes( } } + bool nodeReuseRequested = Handshake.IsHandshakeOptionEnabled(hostHandshake.HandshakeOptions, HandshakeOptions.NodeReuse); // Get all process of possible running node processes for reuse and put them into ConcurrentQueue. // Processes from this queue will be concurrently consumed by TryReusePossibleRunningNodes while // trying to connect to them and reuse them. When queue is empty, no process to reuse left @@ -224,7 +225,7 @@ protected IList GetNodes( ConcurrentQueue possibleRunningNodes = null; #if FEATURE_NODE_REUSE // Try to connect to idle nodes if node reuse is enabled. - if (_componentHost.BuildParameters.EnableNodeReuse) + if (nodeReuseRequested) { IList possibleRunningNodesList; (expectedProcessName, possibleRunningNodesList) = GetPossibleRunningNodes(msbuildLocation); @@ -236,6 +237,7 @@ protected IList GetNodes( } } #endif + ConcurrentQueue nodeContexts = new(); ConcurrentQueue exceptions = new(); int currentProcessId = EnvironmentUtilities.CurrentProcessId; @@ -243,7 +245,12 @@ protected IList GetNodes( { try { - if (!TryReuseAnyFromPossibleRunningNodes(currentProcessId, nodeId) && !StartNewNode(nodeId)) + if (nodeReuseRequested && TryReuseAnyFromPossibleRunningNodes(currentProcessId, nodeId)) + { + return; + } + + if (!StartNewNode(nodeId)) { // We were unable to reuse or launch a node. CommunicationsUtilities.Trace("FAILED TO CONNECT TO A CHILD NODE"); diff --git a/src/Build/Instance/TaskFactories/AssemblyTaskFactory.cs b/src/Build/Instance/TaskFactories/AssemblyTaskFactory.cs index 9da4014e849..9dd3031bdde 100644 --- a/src/Build/Instance/TaskFactories/AssemblyTaskFactory.cs +++ b/src/Build/Instance/TaskFactories/AssemblyTaskFactory.cs @@ -361,10 +361,14 @@ internal ITask CreateTaskInstance( ErrorUtilities.VerifyThrowInternalNull(buildComponentHost); mergedParameters = UpdateTaskHostParameters(mergedParameters); - (mergedParameters, bool isNetRuntime) = AddNetHostParamsIfNeeded(mergedParameters, getProperty); + mergedParameters = AddNetHostParamsIfNeeded(mergedParameters, getProperty); - bool useSidecarTaskHost = !(_factoryIdentityParameters.TaskHostFactoryExplicitlyRequested ?? false) - || isNetRuntime; + // Sidecar here means that the task host is launched with /nodeReuse:true and doesn't terminate + // after the task execution. This improves performance for tasks that run multiple times in a build. + // If the task host factory is explicitly requested, do not act as a sidecar task host. + // This is important as customers use task host factories for short lived tasks to release + // potential locks. + bool useSidecarTaskHost = !(_factoryIdentityParameters.TaskHostFactoryExplicitlyRequested ?? false); TaskHostTask task = new( taskLocation, @@ -631,7 +635,7 @@ private static TaskHostParameters MergeTaskFactoryParameterSets( /// Adds the properties necessary for .NET task host instantiation if the runtime is .NET. /// Returns a new TaskHostParameters with .NET host parameters added, or the original if not needed. /// - private static (TaskHostParameters TaskHostParams, bool isNetRuntime) AddNetHostParamsIfNeeded( + private static TaskHostParameters AddNetHostParamsIfNeeded( in TaskHostParameters currentParams, Func getProperty) { @@ -639,7 +643,7 @@ private static (TaskHostParameters TaskHostParams, bool isNetRuntime) AddNetHost if (currentParams.Runtime == null || !currentParams.Runtime.Equals(XMakeAttributes.MSBuildRuntimeValues.net, StringComparison.OrdinalIgnoreCase)) { - return (currentParams, isNetRuntime: false); + return currentParams; } string dotnetHostPath = getProperty(Constants.DotnetHostPathEnvVarName)?.EvaluatedValue; @@ -647,17 +651,16 @@ private static (TaskHostParameters TaskHostParams, bool isNetRuntime) AddNetHost if (string.IsNullOrEmpty(dotnetHostPath) || string.IsNullOrEmpty(ridGraphPath)) { - return (currentParams, isNetRuntime: false); + return currentParams; } string msBuildAssemblyPath = Path.GetDirectoryName(ridGraphPath) ?? string.Empty; - return (new TaskHostParameters( + return new TaskHostParameters( runtime: currentParams.Runtime, architecture: currentParams.Architecture, dotnetHostPath: dotnetHostPath, - msBuildAssemblyPath: msBuildAssemblyPath), - isNetRuntime: true); + msBuildAssemblyPath: msBuildAssemblyPath); } /// diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 3ebfcae6615..cc492e9b4ed 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -2294,6 +2294,44 @@ public void TestProcessFileLoggerSwitch5() distributedLoggerRecords.Count.ShouldBe(0); // "Expected no distributed loggers to be attached" loggers.Count.ShouldBe(0); // "Expected no central loggers to be attached" } + + /// + /// Verify that DistributedLoggerRecords with null CentralLogger don't cause exceptions when creating ProjectCollection + /// This is a regression test for the issue where -dfl flag caused MSB1025 error due to null logger not being filtered. + /// + [Fact] + public void TestNullCentralLoggerInDistributedLoggerRecord() + { + // Simulate the scenario when using -dfl flag + // ProcessDistributedFileLogger creates a DistributedLoggerRecord with null CentralLogger + var distributedLoggerRecords = new List(); + bool distributedFileLogger = true; + string[] fileLoggerParameters = null; + + MSBuildApp.ProcessDistributedFileLogger( + distributedFileLogger, + fileLoggerParameters, + distributedLoggerRecords); + + // Verify that we have a distributed logger record with null central logger + distributedLoggerRecords.Count.ShouldBe(1); + distributedLoggerRecords[0].CentralLogger.ShouldBeNull(); + + // This should not throw ArgumentNullException when creating ProjectCollection + // The fix filters out null central loggers from the evaluationLoggers array + var loggers = Array.Empty(); + Should.NotThrow(() => + { + using var projectCollection = new ProjectCollection( + new Dictionary(), + loggers: [.. loggers, .. distributedLoggerRecords.Select(d => d.CentralLogger).Where(l => l is not null)], + remoteLoggers: null, + toolsetDefinitionLocations: ToolsetDefinitionLocations.Default, + maxNodeCount: 1, + onlyLogCriticalEvents: false, + loadProjectsReadOnly: true); + }); + } #endregion #region ProcessConsoleLoggerSwitches diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 0bc9bd7aafd..181ae78179a 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1392,8 +1392,8 @@ internal static bool BuildProject( // all of the loggers that are single-node only .. loggers, // all of the central loggers for multi-node systems. These need to be resilient to multiple calls - // to Initialize - .. distributedLoggerRecords.Select(d => d.CentralLogger) + // to Initialize. Filter out null loggers (e.g., DistributedFileLogger uses null central logger). + .. distributedLoggerRecords.Select(d => d.CentralLogger).Where(l => l is not null) ]; projectCollection = new ProjectCollection(