From 083779f84d8f59a1e0e1419ece5beecd959a4030 Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Fri, 25 Jun 2021 11:29:20 -0700 Subject: [PATCH 01/11] Fix missing project instance in project cache requests (#6568) Context Non static graph builds using the project cache didn't set the ProjectInstance on the cache request, leading to crashes in the cache. Changes Made Recreate the BuildRequestData for the cache request after the cache service evaluates projects. I was initially using the original BuildSubmission.BuildRequestData which does not contain the project instance. --- .../ProjectCache/ProjectCacheTests.cs | 82 +++++++++++++++++++ .../ProjectCache/ProjectCacheService.cs | 7 +- .../ProjectCachePlugin/AssemblyMockCache.cs | 4 + .../ProjectCachePlugin.csproj | 3 + 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs index 8e9b451874c..f119d2f50e5 100644 --- a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs +++ b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs @@ -272,6 +272,8 @@ public override async Task GetCacheResultAsync( logger.LogMessage($"MockCache: GetCacheResultAsync for {buildRequest.ProjectFullPath}", MessageImportance.High); + buildRequest.ProjectInstance.ShouldNotBeNull("The cache plugin expects evaluated projects."); + if (_projectQuerySleepTime is not null) { await Task.Delay(_projectQuerySleepTime.Value); @@ -1349,6 +1351,86 @@ Task BuildProjectFileAsync(int projectNumber) } } + [Theory] + [InlineData(false, false)] + // TODO: Reenable when this gets into the main branch. + //[InlineData(true, true)] + public void ParallelStressTestForVsWorkaround(bool useSynchronousLogging, bool disableInprocNode) + { + var currentBuildEnvironment = BuildEnvironmentHelper.Instance; + + try + { + BuildEnvironmentHelper.ResetInstance_ForUnitTestsOnly( + new BuildEnvironment( + currentBuildEnvironment.Mode, + currentBuildEnvironment.CurrentMSBuildExePath, + currentBuildEnvironment.RunningTests, + runningInVisualStudio: true, + visualStudioPath: currentBuildEnvironment.VisualStudioInstallRootDirectory)); + + BuildManager.ProjectCacheItems.ShouldBeEmpty(); + + var referenceNumbers = Enumerable.Range(2, NativeMethodsShared.GetLogicalCoreCount() * 2).ToArray(); + + var testData = new GraphCacheResponse( + new Dictionary + { + {1, referenceNumbers} + }, + referenceNumbers.ToDictionary(k => k, k => GraphCacheResponse.SuccessfulProxyTargetResult()) + ); + + var graph = testData.CreateGraph(_env); + + // Even though the assembly cache is discovered, we'll be overriding it with a descriptor based cache. + BuildManager.ProjectCacheItems.ShouldHaveSingleItem(); + + var cache = new InstanceMockCache(testData, TimeSpan.FromMilliseconds(50)); + + using var buildSession = new Helpers.BuildManagerSession(_env, new BuildParameters + { + MaxNodeCount = NativeMethodsShared.GetLogicalCoreCount(), + ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( + cache, + entryPoints: null, + graph), + UseSynchronousLogging = useSynchronousLogging, + DisableInProcNode = disableInprocNode + }); + + var buildResultTasks = new List>(); + + foreach (var node in graph.ProjectNodes.Where(n => referenceNumbers.Contains(GetProjectNumber(n)))) + { + var buildResultTask = buildSession.BuildProjectFileAsync( + node.ProjectInstance.FullPath, + globalProperties: + new Dictionary { { "SolutionPath", graph.GraphRoots.First().ProjectInstance.FullPath } }); + + buildResultTasks.Add(buildResultTask); + } + + foreach (var buildResultTask in buildResultTasks) + { + buildResultTask.Result.OverallResult.ShouldBe(BuildResultCode.Success); + } + + buildSession.BuildProjectFile( + graph.GraphRoots.First().ProjectInstance.FullPath, + globalProperties: + new Dictionary {{"SolutionPath", graph.GraphRoots.First().ProjectInstance.FullPath}}) + .OverallResult.ShouldBe(BuildResultCode.Success); + + cache.QueryStartStops.Count.ShouldBe(graph.ProjectNodes.Count * 2); + } + finally + { + BuildEnvironmentHelper.ResetInstance_ForUnitTestsOnly(currentBuildEnvironment); + BuildManager.ProjectCacheItems.Clear(); + } + } + [Theory] [InlineData(false, false)] [InlineData(true, true)] diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 8d3484cf052..50fc14a11a0 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -230,7 +230,10 @@ async Task ProcessCacheRequest(CacheRequest request) await LateInitializePluginForVsWorkaround(request); } - return await GetCacheResultAsync(cacheRequest.Submission.BuildRequestData); + return await GetCacheResultAsync( + new BuildRequestData( + request.Configuration.Project, + request.Submission.BuildRequestData.TargetNames.ToArray())); } static bool IsDesignTimeBuild(ProjectInstance project) @@ -300,6 +303,8 @@ static bool MSBuildStringIsTrue(string msbuildString) => private async Task GetCacheResultAsync(BuildRequestData buildRequest) { + ErrorUtilities.VerifyThrowInternalNull(buildRequest.ProjectInstance, nameof(buildRequest.ProjectInstance)); + var queryDescription = $"{buildRequest.ProjectFullPath}" + $"\n\tTargets:[{string.Join(", ", buildRequest.TargetNames)}]" + $"\n\tGlobal Properties: {{{string.Join(",", buildRequest.GlobalProperties.Select(kvp => $"{kvp.Name}={kvp.EvaluatedValue}"))}}}"; diff --git a/src/Samples/ProjectCachePlugin/AssemblyMockCache.cs b/src/Samples/ProjectCachePlugin/AssemblyMockCache.cs index 3b26b82d942..7f049a6c699 100644 --- a/src/Samples/ProjectCachePlugin/AssemblyMockCache.cs +++ b/src/Samples/ProjectCachePlugin/AssemblyMockCache.cs @@ -2,11 +2,13 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Execution; using Microsoft.Build.Experimental.ProjectCache; using Microsoft.Build.Framework; +using Shouldly; namespace MockCacheFromAssembly { @@ -33,6 +35,8 @@ public override Task GetCacheResultAsync( { logger.LogMessage($"{nameof(AssemblyMockCache)}: GetCacheResultAsync for {buildRequest.ProjectFullPath}", MessageImportance.High); + buildRequest.ProjectInstance.ShouldNotBeNull("The cache plugin expects evaluated projects."); + ErrorFrom(nameof(GetCacheResultAsync), logger); return Task.FromResult(CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable)); diff --git a/src/Samples/ProjectCachePlugin/ProjectCachePlugin.csproj b/src/Samples/ProjectCachePlugin/ProjectCachePlugin.csproj index df35ae1ca6b..3e08803f1c5 100644 --- a/src/Samples/ProjectCachePlugin/ProjectCachePlugin.csproj +++ b/src/Samples/ProjectCachePlugin/ProjectCachePlugin.csproj @@ -12,4 +12,7 @@ + + + From 493edc412ced81a17f05be5219a0f02a47423bcd Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Tue, 15 Jun 2021 14:23:40 -0700 Subject: [PATCH 02/11] Don't launch debugger window for all tests --- .../ProjectCache/ProjectCacheTests.cs | 29 ++++++++++--------- .../BackEnd/BuildManager/BuildParameters.cs | 8 +++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs index f119d2f50e5..c69e1872015 100644 --- a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs +++ b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs @@ -31,6 +31,7 @@ public ProjectCacheTests(ITestOutputHelper output) { _output = output; _env = TestEnvironment.Create(output); + _env.DoNotLaunchDebugger(); BuildManager.ProjectCacheItems.ShouldBeEmpty(); _env.WithInvariant(new CustomConditionInvariant(() => BuildManager.ProjectCacheItems.Count == 0)); @@ -432,10 +433,14 @@ public void ProjectCacheByBuildParametersAndGraphBuildWorks(GraphCacheResponse t var graph = testData.CreateGraph(_env); var mockCache = new InstanceMockCache(testData); - buildParameters.ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( + // Reset the environment variables stored in the build params to take into account TestEnvironmentChanges. + buildParameters = new BuildParameters(buildParameters, resetEnvironment: true) + { + ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( mockCache, null, - graph); + graph) + }; using var buildSession = new Helpers.BuildManagerSession(_env, buildParameters); @@ -471,7 +476,12 @@ public void ProjectCacheByBuildParametersAndBottomUpBuildWorks(GraphCacheRespons null, graph); - buildParameters.ProjectCacheDescriptor = projectCacheDescriptor; + // Reset the environment variables stored in the build params to take into account TestEnvironmentChanges. + buildParameters = new BuildParameters(buildParameters, resetEnvironment: true) + { + ProjectCacheDescriptor = projectCacheDescriptor + }; + using var buildSession = new Helpers.BuildManagerSession(_env, buildParameters); var nodesToBuildResults = new Dictionary(); @@ -518,6 +528,9 @@ public void ProjectCacheByVsWorkaroundWorks(GraphCacheResponse testData, BuildPa runningInVisualStudio: true, visualStudioPath: currentBuildEnvironment.VisualStudioInstallRootDirectory)); + // Reset the environment variables stored in the build params to take into account TestEnvironmentChanges. + buildParameters = new BuildParameters(buildParameters, resetEnvironment: true); + BuildManager.ProjectCacheItems.ShouldBeEmpty(); var graph = testData.CreateGraph(_env); @@ -934,8 +947,6 @@ public void BuildFailsWhenCacheBuildResultIsWrong() [Fact] public void GraphBuildErrorsIfMultiplePluginsAreFound() { - _env.DoNotLaunchDebugger(); - var graph = Helpers.CreateProjectGraph( _env, new Dictionary @@ -960,8 +971,6 @@ public void GraphBuildErrorsIfMultiplePluginsAreFound() [Fact] public void GraphBuildErrorsIfNotAllNodeDefineAPlugin() { - _env.DoNotLaunchDebugger(); - var graph = Helpers.CreateProjectGraph( _env, dependencyEdges: new Dictionary @@ -1014,8 +1023,6 @@ public static IEnumerable CacheExceptionLocationsTestData [MemberData(nameof(CacheExceptionLocationsTestData))] public void EngineShouldHandleExceptionsFromCachePluginViaBuildParameters(ErrorLocations errorLocations, ErrorKind errorKind) { - _env.DoNotLaunchDebugger(); - SetEnvironmentForErrorLocations(errorLocations, errorKind.ToString()); var project = _env.CreateFile("1.proj", @$" @@ -1135,8 +1142,6 @@ public void EngineShouldHandleExceptionsFromCachePluginViaBuildParameters(ErrorL [MemberData(nameof(CacheExceptionLocationsTestData))] public void EngineShouldHandleExceptionsFromCachePluginViaGraphBuild(ErrorLocations errorLocations, ErrorKind errorKind) { - _env.DoNotLaunchDebugger(); - SetEnvironmentForErrorLocations(errorLocations, errorKind.ToString()); var graph = Helpers.CreateProjectGraph( @@ -1224,8 +1229,6 @@ public void EngineShouldHandleExceptionsFromCachePluginViaGraphBuild(ErrorLocati [Fact] public void EndBuildShouldGetCalledOnceWhenItThrowsExceptionsFromGraphBuilds() { - _env.DoNotLaunchDebugger(); - var project = _env.CreateFile( "1.proj", @$" diff --git a/src/Build/BackEnd/BuildManager/BuildParameters.cs b/src/Build/BackEnd/BuildManager/BuildParameters.cs index 1259648e255..93d21956172 100644 --- a/src/Build/BackEnd/BuildManager/BuildParameters.cs +++ b/src/Build/BackEnd/BuildManager/BuildParameters.cs @@ -253,7 +253,7 @@ private BuildParameters(ITranslator translator) /// /// Copy constructor /// - private BuildParameters(BuildParameters other) + internal BuildParameters(BuildParameters other, bool resetEnvironment = false) { ErrorUtilities.VerifyThrowInternalNull(other, nameof(other)); @@ -261,7 +261,11 @@ private BuildParameters(BuildParameters other) _culture = other._culture; _defaultToolsVersion = other._defaultToolsVersion; _enableNodeReuse = other._enableNodeReuse; - _buildProcessEnvironment = other._buildProcessEnvironment != null ? new Dictionary(other._buildProcessEnvironment) : null; + _buildProcessEnvironment = resetEnvironment + ? CommunicationsUtilities.GetEnvironmentVariables() + : other._buildProcessEnvironment != null + ? new Dictionary(other._buildProcessEnvironment) + : null; _environmentProperties = other._environmentProperties != null ? new PropertyDictionary(other._environmentProperties) : null; _forwardingLoggers = other._forwardingLoggers != null ? new List(other._forwardingLoggers) : null; _globalProperties = other._globalProperties != null ? new PropertyDictionary(other._globalProperties) : null; From 2880f0aeb934dd94fb1d5011e39720d658d15a95 Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Tue, 15 Jun 2021 14:16:19 -0700 Subject: [PATCH 03/11] Convert static InitializePlugin into non-static BeginBuildAsync To allow asserting service state transition --- .../ProjectCache/ProjectCacheService.cs | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 50fc14a11a0..160da67b809 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -69,39 +69,43 @@ public static async Task FromDescriptorAsync( // their verbosity levels. var loggerFactory = new Func(() => new LoggingServiceToPluginLoggerAdapter(LoggerVerbosity.Normal, loggingService)); - // TODO: remove after we change VS to set the cache descriptor via build parameters. - if (pluginDescriptor.VsWorkaround) - { + var service = new ProjectCacheService(plugin, buildManager, loggerFactory, pluginDescriptor, cancellationToken); + + // TODO: remove the if after we change VS to set the cache descriptor via build parameters and always call BeginBuildAsync in FromDescriptorAsync. // When running under VS we can't initialize the plugin until we evaluate a project (any project) and extract // further information (set by VS) from it required by the plugin. - return new ProjectCacheService(plugin, buildManager, loggerFactory, pluginDescriptor, cancellationToken); + if (!pluginDescriptor.VsWorkaround) + { + await service.BeginBuildAsync(); } - await InitializePlugin(pluginDescriptor, cancellationToken, loggerFactory, plugin); - - return new ProjectCacheService(plugin, buildManager, loggerFactory, pluginDescriptor, cancellationToken); + return service; } - private static async Task InitializePlugin( - ProjectCacheDescriptor pluginDescriptor, - CancellationToken cancellationToken, - Func loggerFactory, - ProjectCachePluginBase plugin - ) + // TODO: remove vsWorkaroundOverrideDescriptor after we change VS to set the cache descriptor via build parameters. + private async Task BeginBuildAsync(ProjectCacheDescriptor? vsWorkaroundOverrideDescriptor = null) { - var logger = loggerFactory(); + var logger = _loggerFactory(); try { - await plugin.BeginBuildAsync( + + if (_projectCacheDescriptor.VsWorkaround) + { + logger.LogMessage("Running project cache with Visual Studio workaround"); + } + + var projectDescriptor = vsWorkaroundOverrideDescriptor ?? _projectCacheDescriptor; + await _projectCachePlugin.BeginBuildAsync( new CacheContext( - pluginDescriptor.PluginSettings, + projectDescriptor.PluginSettings, new IFileSystemAdapter(FileSystems.Default), - pluginDescriptor.ProjectGraph, - pluginDescriptor.EntryPoints), + projectDescriptor.ProjectGraph, + projectDescriptor.EntryPoints), // TODO: Detect verbosity from logging service. logger, - cancellationToken); + _cancellationToken); + } catch (Exception e) { @@ -281,7 +285,7 @@ async Task LateInitializePluginForVsWorkaround(CacheRequest request) FileSystems.Default.FileExists(solutionPath), $"Solution file does not exist: {solutionPath}"); - await InitializePlugin( + await BeginBuildAsync( ProjectCacheDescriptor.FromAssemblyPath( _projectCacheDescriptor.PluginAssemblyPath!, new[] @@ -291,10 +295,7 @@ await InitializePlugin( configuration.Project.GlobalProperties) }, projectGraph: null, - _projectCacheDescriptor.PluginSettings), - _cancellationToken, - _loggerFactory, - _projectCachePlugin); + _projectCacheDescriptor.PluginSettings)); } static bool MSBuildStringIsTrue(string msbuildString) => From d2b31c29945094a8044441745a9090bc196b348d Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Tue, 15 Jun 2021 14:25:30 -0700 Subject: [PATCH 04/11] Assert state transitions in ProjectCacheService --- .../ProjectCache/ProjectCacheTests.cs | 19 ++-- .../ProjectCache/ProjectCacheService.cs | 89 +++++++++++++++++-- src/Shared/ErrorUtilities.cs | 8 ++ 3 files changed, 100 insertions(+), 16 deletions(-) diff --git a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs index c69e1872015..dca89881a97 100644 --- a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs +++ b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs @@ -437,8 +437,8 @@ public void ProjectCacheByBuildParametersAndGraphBuildWorks(GraphCacheResponse t buildParameters = new BuildParameters(buildParameters, resetEnvironment: true) { ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( - mockCache, - null, + mockCache, + null, graph) }; @@ -553,6 +553,7 @@ public void ProjectCacheByVsWorkaroundWorks(GraphCacheResponse testData, BuildPa } buildSession.Logger.FullLog.ShouldContain("Visual Studio Workaround based"); + buildSession.Logger.FullLog.ShouldContain("Running project cache with Visual Studio workaround"); AssertCacheBuild(graph, testData, null, buildSession.Logger, nodesToBuildResults); } @@ -618,6 +619,9 @@ public void DesignTimeBuildsDuringVsWorkaroundShouldDisableTheCache() buildSession.Logger.FullLog.ShouldContain("Visual Studio Workaround based"); + // Design time builds should not initialize the plugin. + buildSession.Logger.FullLog.ShouldNotContain("Running project cache with Visual Studio workaround"); + // Cache doesn't get initialized and queried. buildSession.Logger.FullLog.ShouldNotContain("BeginBuildAsync"); buildSession.Logger.FullLog.ShouldNotContain("GetCacheResultAsync for"); @@ -1389,15 +1393,9 @@ public void ParallelStressTestForVsWorkaround(bool useSynchronousLogging, bool d // Even though the assembly cache is discovered, we'll be overriding it with a descriptor based cache. BuildManager.ProjectCacheItems.ShouldHaveSingleItem(); - var cache = new InstanceMockCache(testData, TimeSpan.FromMilliseconds(50)); - using var buildSession = new Helpers.BuildManagerSession(_env, new BuildParameters { MaxNodeCount = NativeMethodsShared.GetLogicalCoreCount(), - ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( - cache, - entryPoints: null, - graph), UseSynchronousLogging = useSynchronousLogging, DisableInProcNode = disableInprocNode }); @@ -1425,7 +1423,10 @@ public void ParallelStressTestForVsWorkaround(bool useSynchronousLogging, bool d new Dictionary {{"SolutionPath", graph.GraphRoots.First().ProjectInstance.FullPath}}) .OverallResult.ShouldBe(BuildResultCode.Success); - cache.QueryStartStops.Count.ShouldBe(graph.ProjectNodes.Count * 2); + StringShouldContainSubstring(buildSession.Logger.FullLog, $"{AssemblyMockCache}: GetCacheResultAsync for", graph.ProjectNodes.Count); + + buildSession.Logger.FullLog.ShouldContain("Visual Studio Workaround based"); + buildSession.Logger.FullLog.ShouldContain("Running project cache with Visual Studio workaround"); } finally { diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 160da67b809..1e0e3ee67ec 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -27,6 +27,15 @@ internal record NullableBool(bool Value) public static implicit operator bool(NullableBool? d) => d is not null && d.Value; } + internal enum ProjectCacheServiceState + { + NotInitialized, + BeginBuildStarted, + BeginBuildFinished, + ShutdownStarted, + ShutdownFinished + } + internal class ProjectCacheService { private readonly BuildManager _buildManager; @@ -34,6 +43,7 @@ internal class ProjectCacheService private readonly ProjectCacheDescriptor _projectCacheDescriptor; private readonly CancellationToken _cancellationToken; private readonly ProjectCachePluginBase _projectCachePlugin; + private ProjectCacheServiceState _serviceState = ProjectCacheServiceState.NotInitialized; // Use NullableBool to make it work with Interlock.CompareExchange (doesn't accept bool?). // Assume that if one request is a design time build, all of them are. @@ -72,8 +82,8 @@ public static async Task FromDescriptorAsync( var service = new ProjectCacheService(plugin, buildManager, loggerFactory, pluginDescriptor, cancellationToken); // TODO: remove the if after we change VS to set the cache descriptor via build parameters and always call BeginBuildAsync in FromDescriptorAsync. - // When running under VS we can't initialize the plugin until we evaluate a project (any project) and extract - // further information (set by VS) from it required by the plugin. + // When running under VS we can't initialize the plugin until we evaluate a project (any project) and extract + // further information (set by VS) from it required by the plugin. if (!pluginDescriptor.VsWorkaround) { await service.BeginBuildAsync(); @@ -89,6 +99,7 @@ private async Task BeginBuildAsync(ProjectCacheDescriptor? vsWorkaroundOverrideD try { + SetState(ProjectCacheServiceState.BeginBuildStarted); if (_projectCacheDescriptor.VsWorkaround) { @@ -106,6 +117,7 @@ await _projectCachePlugin.BeginBuildAsync( logger, _cancellationToken); + SetState(ProjectCacheServiceState.BeginBuildFinished); } catch (Exception e) { @@ -230,7 +242,7 @@ async Task ProcessCacheRequest(CacheRequest request) if (_projectCacheDescriptor.VsWorkaround) { - // TODO: remove after we change VS to set the cache descriptor via build parameters. + // TODO: remove after we change VS to set the cache descriptor via build parameters. await LateInitializePluginForVsWorkaround(request); } @@ -304,6 +316,17 @@ static bool MSBuildStringIsTrue(string msbuildString) => private async Task GetCacheResultAsync(BuildRequestData buildRequest) { + lock (this) + { + CheckNotInState(ProjectCacheServiceState.NotInitialized); + CheckNotInState(ProjectCacheServiceState.BeginBuildStarted); + + if (_serviceState is ProjectCacheServiceState.ShutdownStarted or ProjectCacheServiceState.ShutdownFinished) + { + return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); + } + } + ErrorUtilities.VerifyThrowInternalNull(buildRequest.ProjectInstance, nameof(buildRequest.ProjectInstance)); var queryDescription = $"{buildRequest.ProjectFullPath}" + @@ -361,16 +384,22 @@ public async Task ShutDown() try { + SetState(ProjectCacheServiceState.ShutdownStarted); + await _projectCachePlugin.EndBuildAsync(logger, _cancellationToken); + + if (logger.HasLoggedErrors) + { + ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheShutdownFailed"); + } } - catch (Exception e) + catch (Exception e) when (e is not ProjectCacheException) { HandlePluginException(e, nameof(ProjectCachePluginBase.EndBuildAsync)); } - - if (logger.HasLoggedErrors) + finally { - ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheShutdownFailed"); + SetState(ProjectCacheServiceState.ShutdownFinished); } } @@ -387,6 +416,52 @@ private static void HandlePluginException(Exception e, string apiExceptionWasThr apiExceptionWasThrownFrom); } + private void SetState(ProjectCacheServiceState newState) + { + lock (this) + { + switch (newState) + { + case ProjectCacheServiceState.NotInitialized: + ErrorUtilities.ThrowInternalError($"Cannot transition to {ProjectCacheServiceState.NotInitialized}"); + break; + case ProjectCacheServiceState.BeginBuildStarted: + CheckInState(ProjectCacheServiceState.NotInitialized); + break; + case ProjectCacheServiceState.BeginBuildFinished: + CheckInState(ProjectCacheServiceState.BeginBuildStarted); + break; + case ProjectCacheServiceState.ShutdownStarted: + CheckNotInState(ProjectCacheServiceState.ShutdownStarted); + CheckNotInState(ProjectCacheServiceState.ShutdownFinished); + break; + case ProjectCacheServiceState.ShutdownFinished: + CheckInState(ProjectCacheServiceState.ShutdownStarted); + break; + default: + throw new ArgumentOutOfRangeException(nameof(newState), newState, null); + } + + _serviceState = newState; + } + } + + private void CheckInState(ProjectCacheServiceState expectedState) + { + lock (this) + { + ErrorUtilities.VerifyThrowInternalError(_serviceState == expectedState, $"Expected state {expectedState}, actual state {_serviceState}"); + } + } + + private void CheckNotInState(ProjectCacheServiceState unexpectedState) + { + lock (this) + { + ErrorUtilities.VerifyThrowInternalError(_serviceState != unexpectedState, $"Unexpected state {_serviceState}"); + } + } + private class LoggingServiceToPluginLoggerAdapter : PluginLoggerBase { private readonly ILoggingService _loggingService; diff --git a/src/Shared/ErrorUtilities.cs b/src/Shared/ErrorUtilities.cs index 2731c90b61a..9bb3502596b 100644 --- a/src/Shared/ErrorUtilities.cs +++ b/src/Shared/ErrorUtilities.cs @@ -47,6 +47,14 @@ public static void DebugTraceMessage(string category, string formatstring, param #if !BUILDINGAPPXTASKS #region VerifyThrow -- for internal errors + internal static void VerifyThrowInternalError(bool condition, string message, params object[] args) + { + if (s_throwExceptions && !condition) + { + throw new InternalErrorException(ResourceUtilities.FormatString(message, args)); + } + } + /// /// Throws InternalErrorException. /// This is only for situations that would mean that there is a bug in MSBuild itself. From ad5f2fa5ff858b3a63f02a15c2ce6a6024e3d30e Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Tue, 15 Jun 2021 16:05:47 -0700 Subject: [PATCH 05/11] Only initialize once for the VS workaround --- .../ProjectCache/ProjectCacheService.cs | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 1e0e3ee67ec..582db7d0da9 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -50,6 +50,7 @@ internal class ProjectCacheService // Volatile because it is read by the BuildManager thread and written by one project cache service thread pool thread. // TODO: remove after we change VS to set the cache descriptor via build parameters. public volatile NullableBool? DesignTimeBuildsDetected; + private TaskCompletionSource? LateInitializationForVSWorkaroundCompleted; private ProjectCacheService( ProjectCachePluginBase projectCachePlugin, @@ -225,13 +226,20 @@ async Task ProcessCacheRequest(CacheRequest request) EvaluateProjectIfNecessary(request); + // Detect design time builds. if (_projectCacheDescriptor.VsWorkaround) { - Interlocked.CompareExchange( + var isDesignTimeBuild = IsDesignTimeBuild(request.Configuration.Project); + + var previousValue = Interlocked.CompareExchange( ref DesignTimeBuildsDetected, - new NullableBool(IsDesignTimeBuild(request.Configuration.Project)), + new NullableBool(isDesignTimeBuild), null); + ErrorUtilities.VerifyThrowInternalError( + previousValue is null || previousValue == false || isDesignTimeBuild, + "Either all builds in a build session or design time builds, or none"); + // No point progressing with expensive plugin initialization or cache query if design time build detected. if (DesignTimeBuildsDetected) { @@ -240,11 +248,28 @@ async Task ProcessCacheRequest(CacheRequest request) } } - if (_projectCacheDescriptor.VsWorkaround) - { // TODO: remove after we change VS to set the cache descriptor via build parameters. + // VS workaround needs to wait until the first project is evaluated to extract enough information to initialize the plugin. + // No cache request can progress until late initialization is complete. + if (_projectCacheDescriptor.VsWorkaround && + Interlocked.CompareExchange( + ref LateInitializationForVSWorkaroundCompleted, + new TaskCompletionSource(), + null) is null) + { await LateInitializePluginForVsWorkaround(request); + LateInitializationForVSWorkaroundCompleted.SetResult(true); } + else if (_projectCacheDescriptor.VsWorkaround) + { + // Can't be null. If the thread got here it means another thread initialized the completion source. + await LateInitializationForVSWorkaroundCompleted!.Task; + } + + ErrorUtilities.VerifyThrowInternalError( + LateInitializationForVSWorkaroundCompleted is null || + _projectCacheDescriptor.VsWorkaround && LateInitializationForVSWorkaroundCompleted.Task.IsCompleted, + "Completion source should be null when this is not the VS workaround"); return await GetCacheResultAsync( new BuildRequestData( From 5e2f4db492eafbeb8d7969680b34b7f36ba60e64 Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Fri, 18 Jun 2021 10:37:36 -0700 Subject: [PATCH 06/11] Bravely set DoNotLaunchDebugger only once for all tests --- .../BackEnd/BuildRequestConfiguration_Tests.cs | 1 - .../Definition/ProjectEvaluationContext_Tests.cs | 2 -- .../Graph/GraphLoadedFromSolution_tests.cs | 8 -------- src/Build.UnitTests/Graph/ProjectGraph_Tests.cs | 1 - src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs | 1 - src/Shared/UnitTests/TestAssemblyInfo.cs | 2 ++ 6 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/Build.UnitTests/BackEnd/BuildRequestConfiguration_Tests.cs b/src/Build.UnitTests/BackEnd/BuildRequestConfiguration_Tests.cs index 5be64a17d58..085f3488877 100644 --- a/src/Build.UnitTests/BackEnd/BuildRequestConfiguration_Tests.cs +++ b/src/Build.UnitTests/BackEnd/BuildRequestConfiguration_Tests.cs @@ -26,7 +26,6 @@ public class BuildRequestConfiguration_Tests : IDisposable public BuildRequestConfiguration_Tests(ITestOutputHelper testOutput) { _env = TestEnvironment.Create(testOutput); - _env.DoNotLaunchDebugger(); } public void Dispose() diff --git a/src/Build.UnitTests/Definition/ProjectEvaluationContext_Tests.cs b/src/Build.UnitTests/Definition/ProjectEvaluationContext_Tests.cs index 00dc1bb6f61..4208ae11b97 100644 --- a/src/Build.UnitTests/Definition/ProjectEvaluationContext_Tests.cs +++ b/src/Build.UnitTests/Definition/ProjectEvaluationContext_Tests.cs @@ -125,8 +125,6 @@ public void PassedInFileSystemShouldBeReusedInSharedContext() [Fact] public void IsolatedContextShouldNotSupportBeingPassedAFileSystem() { - _env.DoNotLaunchDebugger(); - var fileSystem = new Helpers.LoggingFileSystem(); Should.Throw(() => EvaluationContext.Create(EvaluationContext.SharingPolicy.Isolated, fileSystem)); } diff --git a/src/Build.UnitTests/Graph/GraphLoadedFromSolution_tests.cs b/src/Build.UnitTests/Graph/GraphLoadedFromSolution_tests.cs index c1f0161e91d..bed09d043ec 100644 --- a/src/Build.UnitTests/Graph/GraphLoadedFromSolution_tests.cs +++ b/src/Build.UnitTests/Graph/GraphLoadedFromSolution_tests.cs @@ -33,8 +33,6 @@ public GraphLoadedFromSolutionTests(ITestOutputHelper output) [InlineData("1.sln", "2.proj")] public void ASolutionShouldBeTheSingleEntryPoint(params string[] files) { - _env.DoNotLaunchDebugger(); - for (var i = 0; i < files.Length; i++) { files[i] = _env.CreateFile(files[i], string.Empty).Path; @@ -52,8 +50,6 @@ public void ASolutionShouldBeTheSingleEntryPoint(params string[] files) [Fact] public void GraphConstructionFailsOnNonExistentSolution() { - _env.DoNotLaunchDebugger(); - var exception = Should.Throw( () => { @@ -80,8 +76,6 @@ public void StaticGraphShouldNotSupportNestedSolutions() defaultTargets: null, extraContent: referenceToSolution); - _env.DoNotLaunchDebugger(); - var exception = Should.Throw( () => { @@ -621,8 +615,6 @@ IEnumerable GetIncomingEdgeItemsToNode(ProjectGraphNode nod [Fact] public void GraphConstructionShouldThrowOnMissingSolutionDependencies() { - _env.DoNotLaunchDebugger(); - var solutionContents = SolutionFileBuilder.FromGraphEdges( _env, new Dictionary {{1, null}, {2, null}}, diff --git a/src/Build.UnitTests/Graph/ProjectGraph_Tests.cs b/src/Build.UnitTests/Graph/ProjectGraph_Tests.cs index 548a25b3858..609f24fac92 100644 --- a/src/Build.UnitTests/Graph/ProjectGraph_Tests.cs +++ b/src/Build.UnitTests/Graph/ProjectGraph_Tests.cs @@ -106,7 +106,6 @@ public void ConstructWithSingleNodeWithProjectInstanceFactory() [Fact] public void ProjectGraphNodeConstructorNoNullArguments() { - _env.DoNotLaunchDebugger(); Assert.Throws(() => new ProjectGraphNode(null)); } diff --git a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs index dca89881a97..65b60c15a0c 100644 --- a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs +++ b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs @@ -31,7 +31,6 @@ public ProjectCacheTests(ITestOutputHelper output) { _output = output; _env = TestEnvironment.Create(output); - _env.DoNotLaunchDebugger(); BuildManager.ProjectCacheItems.ShouldBeEmpty(); _env.WithInvariant(new CustomConditionInvariant(() => BuildManager.ProjectCacheItems.Count == 0)); diff --git a/src/Shared/UnitTests/TestAssemblyInfo.cs b/src/Shared/UnitTests/TestAssemblyInfo.cs index e1e7ef66d5a..627aa0d465e 100644 --- a/src/Shared/UnitTests/TestAssemblyInfo.cs +++ b/src/Shared/UnitTests/TestAssemblyInfo.cs @@ -35,6 +35,8 @@ public MSBuildTestAssemblyFixture() _testEnvironment = TestEnvironment.Create(); + _testEnvironment.DoNotLaunchDebugger(); + // Reset the VisualStudioVersion environment variable. This will be set if tests are run from a VS command prompt. However, // if the environment variable is set, it will interfere with tests which set the SubToolsetVersion // (VerifySubToolsetVersionSetByConstructorOverridable), as the environment variable would take precedence. From a365fbf2315b22bd46155a60063bdb02fec1831c Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Fri, 18 Jun 2021 10:47:29 -0700 Subject: [PATCH 07/11] Simplify branching --- .../ProjectCache/ProjectCacheService.cs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 582db7d0da9..e5e25ce697e 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -251,19 +251,21 @@ async Task ProcessCacheRequest(CacheRequest request) // TODO: remove after we change VS to set the cache descriptor via build parameters. // VS workaround needs to wait until the first project is evaluated to extract enough information to initialize the plugin. // No cache request can progress until late initialization is complete. - if (_projectCacheDescriptor.VsWorkaround && - Interlocked.CompareExchange( - ref LateInitializationForVSWorkaroundCompleted, - new TaskCompletionSource(), - null) is null) - { - await LateInitializePluginForVsWorkaround(request); - LateInitializationForVSWorkaroundCompleted.SetResult(true); - } - else if (_projectCacheDescriptor.VsWorkaround) + if (_projectCacheDescriptor.VsWorkaround) { - // Can't be null. If the thread got here it means another thread initialized the completion source. - await LateInitializationForVSWorkaroundCompleted!.Task; + if (Interlocked.CompareExchange( + ref LateInitializationForVSWorkaroundCompleted, + new TaskCompletionSource(), + null) is null) + { + await LateInitializePluginForVsWorkaround(request); + LateInitializationForVSWorkaroundCompleted.SetResult(true); + } + else + { + // Can't be null. If the thread got here it means another thread initialized the completion source. + await LateInitializationForVSWorkaroundCompleted!.Task; + } } ErrorUtilities.VerifyThrowInternalError( From bbde2b8c40d1daf14f1ebdd5f404f588c70506af Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 30 Jun 2021 19:00:46 +0200 Subject: [PATCH 08/11] [vs16.11] Update dependencies from dotnet/arcade (#6625) * Update dependencies from https://github.com/dotnet/arcade build 20210628.3 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.21315.2 -> To Version 5.0.0-beta.21328.3 --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 13 +++++++++++-- global.json | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cde22e5ffed..3729b669c02 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - a5dbede4615c46dfb68a894bf090cf517f87efc9 + 5266aa9856457785b84739fda2616f21da7ee6b4 https://github.com/nuget/nuget.client diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60eb601c8f3..eba7ed49d78 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -371,7 +371,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + + $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" + $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false } + if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { + $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" + } else { + $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" + } + + return $global:_MSBuildExe } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { @@ -527,7 +536,7 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build. + # Use local cache on CI to ensure deterministic build. # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 diff --git a/global.json b/global.json index aab18f498ad..f2cc115408c 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.21315.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.21328.3" } } From 1d7ed8e3d8394d0effaee6d3395c28d9322aa9bb Mon Sep 17 00:00:00 2001 From: Mihai Codoban Date: Thu, 1 Jul 2021 11:17:53 -0700 Subject: [PATCH 09/11] Don't schedule proxy builds to inproc node if their configs previously built on oop nodes (#6635) Fixes a bug in proxy build scheduling introduced by #6386. If a the BuildRequestConfiguration associated with a proxy request has been built before on an out of proc (oop) node then the scheduler will fail with either one of: - affinity mismatch error. This happens when the proxy build is assigned to the inproc (inp) node but its configuration is already assigned to an oop node `AND` serving other existing requests, either blocked or running. - unscheduled requests remain even if there's free oop nodes that can serve them. This happens (as far as I can tell) when the proxy's configuration is already assigned to an oop node (because a previously built non proxy request was assigned there) `AND` there's no other existing requests for that configuration The fix in this PR is to not assign a proxy build to the inproc node if its configuration was previously assigned to another node. --- .../ProjectCache/ProjectCacheTests.cs | 115 ++++++++++++++++++ .../ProjectCache/ProjectCacheService.cs | 13 ++ .../BackEnd/Components/Scheduler/Scheduler.cs | 8 +- .../Components/Scheduler/SchedulingData.cs | 7 +- 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs index 65b60c15a0c..79446a04695 100644 --- a/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs +++ b/src/Build.UnitTests/ProjectCache/ProjectCacheTests.cs @@ -233,6 +233,30 @@ public enum ErrorKind LoggedError } + public class ConfigurableMockCache : ProjectCachePluginBase + { + public Func>? GetCacheResultImplementation { get; set; } + public override Task BeginBuildAsync(CacheContext context, PluginLoggerBase logger, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public override Task GetCacheResultAsync( + BuildRequestData buildRequest, + PluginLoggerBase logger, + CancellationToken cancellationToken) + { + return GetCacheResultImplementation != null + ? GetCacheResultImplementation(buildRequest, logger, cancellationToken) + : Task.FromResult(CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable)); + } + + public override Task EndBuildAsync(PluginLoggerBase logger, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + public class InstanceMockCache : ProjectCachePluginBase { private readonly GraphCacheResponse? _testData; @@ -1475,6 +1499,97 @@ public void ParallelStressTest(bool useSynchronousLogging, bool disableInprocNod cache.QueryStartStops.Count.ShouldBe(graph.ProjectNodes.Count * 2); } + [Fact] + // Schedules different requests for the same BuildRequestConfiguration in parallel. + // The first batch of the requests are cache misses, the second batch are cache hits via proxy builds. + // The first batch is delayed so it starts intermingling with the second batch. + // This test ensures that scheduling proxy builds on the inproc node works nicely within the Scheduler + // if the BuildRequestConfigurations for those proxy builds have built before (or are still building) on + // the out of proc node. + // More details: https://github.com/dotnet/msbuild/pull/6635 + public void ProxyCacheHitsOnPreviousCacheMissesShouldWork() + { + var cacheNotApplicableTarget = "NATarget"; + var cacheHitTarget = "CacheHitTarget"; + var proxyTarget = "ProxyTarget"; + + var project = +@$" + + + + + + + + + + + + + + +".Cleanup(); + + var projectPaths = Enumerable.Range(0, NativeMethodsShared.GetLogicalCoreCount()) + .Select(i => _env.CreateFile($"project{i}.proj", project).Path) + .ToArray(); + + var cacheHitCount = 0; + var nonCacheHitCount = 0; + + var buildParameters = new BuildParameters + { + ProjectCacheDescriptor = ProjectCacheDescriptor.FromInstance( + new ConfigurableMockCache + { + GetCacheResultImplementation = (request, _, _) => + { + var projectFile = request.ProjectFullPath; + + if (request.TargetNames.Contains(cacheNotApplicableTarget)) + { + Interlocked.Increment(ref nonCacheHitCount); + return Task.FromResult(CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable)); + } + else + { + Interlocked.Increment(ref cacheHitCount); + return Task.FromResult( + CacheResult.IndicateCacheHit( + new ProxyTargets(new Dictionary {{proxyTarget, cacheHitTarget}}))); + } + } + }, + projectPaths.Select(p => new ProjectGraphEntryPoint(p)).ToArray(), + projectGraph: null), + MaxNodeCount = NativeMethodsShared.GetLogicalCoreCount() + }; + + using var buildSession = new Helpers.BuildManagerSession(_env, buildParameters); + + var buildRequests = new List<(string, string)>(); + buildRequests.AddRange(projectPaths.Select(r => (r, cacheNotApplicableTarget))); + buildRequests.AddRange(projectPaths.Select(r => (r, cacheHitTarget))); + + var buildTasks = new List>(); + foreach (var (projectPath, target) in buildRequests) + { + buildTasks.Add(buildSession.BuildProjectFileAsync(projectPath, new[] {target})); + } + + foreach (var buildResult in buildTasks.Select(buildTask => buildTask.Result)) + { + buildResult.Exception.ShouldBeNull(); + buildResult.OverallResult.ShouldBe(BuildResultCode.Success); + } + + buildSession.Logger.ProjectStartedEvents.Count.ShouldBe(2 * projectPaths.Length); + + cacheHitCount.ShouldBe(projectPaths.Length); + nonCacheHitCount.ShouldBe(projectPaths.Length); + } + private static void StringShouldContainSubstring(string aString, string substring, int expectedOccurrences) { aString.ShouldContain(substring); diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index e5e25ce697e..7d7c0dc5759 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -4,6 +4,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Threading; @@ -102,6 +103,9 @@ private async Task BeginBuildAsync(ProjectCacheDescriptor? vsWorkaroundOverrideD { SetState(ProjectCacheServiceState.BeginBuildStarted); + logger.LogMessage("Initializing project cache plugin", MessageImportance.Low); + var timer = Stopwatch.StartNew(); + if (_projectCacheDescriptor.VsWorkaround) { logger.LogMessage("Running project cache with Visual Studio workaround"); @@ -118,6 +122,9 @@ await _projectCachePlugin.BeginBuildAsync( logger, _cancellationToken); + timer.Stop(); + logger.LogMessage($"Finished initializing project cache plugin in {timer.Elapsed.TotalMilliseconds} ms", MessageImportance.Low); + SetState(ProjectCacheServiceState.BeginBuildFinished); } catch (Exception e) @@ -413,8 +420,14 @@ public async Task ShutDown() { SetState(ProjectCacheServiceState.ShutdownStarted); + logger.LogMessage("Shutting down project cache plugin", MessageImportance.Low); + var timer = Stopwatch.StartNew(); + await _projectCachePlugin.EndBuildAsync(logger, _cancellationToken); + timer.Stop(); + logger.LogMessage($"Finished shutting down project cache plugin in {timer.Elapsed.TotalMilliseconds} ms", MessageImportance.Low); + if (logger.HasLoggedErrors) { ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheShutdownFailed"); diff --git a/src/Build/BackEnd/Components/Scheduler/Scheduler.cs b/src/Build/BackEnd/Components/Scheduler/Scheduler.cs index 03fc8f8a1ab..e95847208e8 100644 --- a/src/Build/BackEnd/Components/Scheduler/Scheduler.cs +++ b/src/Build/BackEnd/Components/Scheduler/Scheduler.cs @@ -1385,7 +1385,7 @@ private void AssignUnscheduledRequestToNode(SchedulableRequest request, int node void WarnWhenProxyBuildsGetScheduledOnOutOfProcNode() { - if (request.IsProxyBuildRequest() && nodeId != InProcNodeId) + if (request.IsProxyBuildRequest() && nodeId != InProcNodeId && _schedulingData.CanScheduleRequestToNode(request, InProcNodeId)) { ErrorUtilities.VerifyThrow( _componentHost.BuildParameters.DisableInProcNode || _forceAffinityOutOfProc, @@ -2107,7 +2107,11 @@ private NodeAffinity GetNodeAffinityForRequest(BuildRequest request) return NodeAffinity.InProc; } - if (request.IsProxyBuildRequest()) + ErrorUtilities.VerifyThrow(request.ConfigurationId != BuildRequestConfiguration.InvalidConfigurationId, "Requests should have a valid configuration id at this point"); + // If this configuration has been previously built on an out of proc node, scheduling it on the inproc node can cause either an affinity mismatch error when + // there are other pending requests for the same configuration or "unscheduled requests remain in the presence of free out of proc nodes" errors if there's no pending requests. + // So only assign proxy builds to the inproc node if their config hasn't been previously assigned to an out of proc node. + if (_schedulingData.CanScheduleConfigurationToNode(request.ConfigurationId, InProcNodeId) && request.IsProxyBuildRequest()) { return NodeAffinity.InProc; } diff --git a/src/Build/BackEnd/Components/Scheduler/SchedulingData.cs b/src/Build/BackEnd/Components/Scheduler/SchedulingData.cs index 0edc83f296e..9aeb9009c80 100644 --- a/src/Build/BackEnd/Components/Scheduler/SchedulingData.cs +++ b/src/Build/BackEnd/Components/Scheduler/SchedulingData.cs @@ -647,7 +647,12 @@ public int GetAssignedNodeForRequestConfiguration(int configurationId) /// public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId) { - int requiredNodeId = GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId); + return CanScheduleConfigurationToNode(request.BuildRequest.ConfigurationId, nodeId); + } + + public bool CanScheduleConfigurationToNode(int configurationId, int nodeId) + { + int requiredNodeId = GetAssignedNodeForRequestConfiguration(configurationId); return requiredNodeId == Scheduler.InvalidNodeId || requiredNodeId == nodeId; } From 07a3cbdbbdc6d642987cbda50e7d9d63710e58c0 Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Fri, 2 Jul 2021 04:13:01 -0700 Subject: [PATCH 10/11] Localized file check-in by OneLocBuild Task (#6644) --- src/MSBuild/Resources/xlf/Strings.cs.xlf | 10 +++++----- src/MSBuild/Resources/xlf/Strings.de.xlf | 2 +- src/MSBuild/Resources/xlf/Strings.fr.xlf | 11 +++++------ src/MSBuild/Resources/xlf/Strings.it.xlf | 6 ++++-- src/MSBuild/Resources/xlf/Strings.pl.xlf | 5 +++-- src/MSBuild/Resources/xlf/Strings.pt-BR.xlf | 9 ++++++--- src/MSBuild/Resources/xlf/Strings.ru.xlf | 13 ++++++------- src/MSBuild/Resources/xlf/Strings.tr.xlf | 6 +++--- src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.zh-Hans.xlf | 4 ++-- 10 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/MSBuild/Resources/xlf/Strings.cs.xlf b/src/MSBuild/Resources/xlf/Strings.cs.xlf index b34392b1273..5769d82fa59 100644 --- a/src/MSBuild/Resources/xlf/Strings.cs.xlf +++ b/src/MSBuild/Resources/xlf/Strings.cs.xlf @@ -494,10 +494,10 @@ Copyright (C) Microsoft Corporation. Všechna práva vyhrazena. ErrorsOnly – zobrazí jenom chyby. WarningsOnly – zobrazí jenom upozornění. NoItemAndPropertyList – nezobrazí na začátku sestavení každého - projektu seznamy položek a vlastností. - ShowCommandLine – zobrazí zprávy TaskCommandLineEvent. + projektu seznamy položek a vlastností. + ShowCommandLine – zobrazí zprávy TaskCommandLineEvent. ShowTimestamp – před každou zprávou zobrazí - časové razítko. + časové razítko. ShowEventId – zobrazí ID události pro spuštěné a dokončené události a zprávy. ForceNoAlign – nenastavuje text podle velikosti vyrovnávací @@ -634,7 +634,7 @@ Copyright (C) Microsoft Corporation. Všechna práva vyhrazena. Setting this also turns on isolated builds (-isolate). (short form: -orc) - -outputResultsCache:<souborMezipaměti>... + -outputResultsCache:[souborMezipaměti]... Výstupní soubor mezipaměti, do něhož bude MSBuild zapisovat obsah svých mezipamětí výsledků sestavení. Nastavením této možnosti zapnete také izolované buildy (-isolate). @@ -794,7 +794,7 @@ Copyright (C) Microsoft Corporation. Všechna práva vyhrazena. template and append the node id to this fileName to create a log file for each node. - -distributedFileLogger + -distributedFileLogger Uloží výstup sestavení do více souborů protokolu, po jednom pro každý uzel nástroje MSBuild. Tyto soubory jsou na počátku umístěny v aktuálním adresáři. Standardně mají tyto soubory diff --git a/src/MSBuild/Resources/xlf/Strings.de.xlf b/src/MSBuild/Resources/xlf/Strings.de.xlf index 140c72a56e9..9ccad6d1801 100644 --- a/src/MSBuild/Resources/xlf/Strings.de.xlf +++ b/src/MSBuild/Resources/xlf/Strings.de.xlf @@ -505,7 +505,7 @@ Copyright (C) Microsoft Corporation. Alle Rechte vorbehalten. bei der Mehrprozessorprotokollierung im Modus mit nur einem Prozessor. EnableMPLogging: Aktiviert das Format der Mehrprozessorprotokollierung auch bei der Ausführung - im Modus mit nur einem Prozessor. Dieses Protokollierungsformat ist standardmäßig aktiviert. + im Modus mit nur einem Prozessor. Dieses Protokollierungsformat ist standardmäßig aktiviert. ForceConsoleColor: Verwendet selbst dann ANSI-Konsolenfarben, wenn die Konsole dies nicht unterstützt. diff --git a/src/MSBuild/Resources/xlf/Strings.fr.xlf b/src/MSBuild/Resources/xlf/Strings.fr.xlf index 86f63c97eb7..ecd04ae29ab 100644 --- a/src/MSBuild/Resources/xlf/Strings.fr.xlf +++ b/src/MSBuild/Resources/xlf/Strings.fr.xlf @@ -844,13 +844,12 @@ Copyright (C) Microsoft Corporation. Tous droits réservés. Paramètres supplémentaires pour les journaliseurs de fichiers. La présence de ce commutateur implique l'utilisation du commutateur -fileLogger[n] correspondant. - S'il est spécifié, "n" doit être un chiffre entre 1 et 9. + S'il est spécifié, "n" doit être un chiffre entre 1 et 9. -fileLoggerParameters est également utilisé par tous les - journaliseurs vers des fichiers. Consultez la description - de -distributedFileLogger. - (Forme abrégée : -flp[n]) + journaliseurs vers des fichiers. Consultez la description de -distributedFileLogger. + (Forme abrégée : -flp[n]) Les mêmes paramètres que ceux listés pour le journaliseur de la - console sont disponibles. Paramètres supplémentaires disponibles : + console sont disponibles. Paramètres supplémentaires disponibles : LogFile--Chemin du fichier journal dans lequel le journal de génération est écrit. Append--Détermine si le journal de génération est ajouté @@ -862,7 +861,7 @@ Copyright (C) Microsoft Corporation. Tous droits réservés. Encoding--Spécifie l'encodage du fichier, par exemple, UTF-8, Unicode ou ASCII Le niveau de détail par défaut est Detailed. - Exemples : + Exemples : -fileLoggerParameters:LogFile=MyLog.log;Append; Verbosity=diagnostic;Encoding=UTF-8 diff --git a/src/MSBuild/Resources/xlf/Strings.it.xlf b/src/MSBuild/Resources/xlf/Strings.it.xlf index 7dd1902d871..c37e60c4dc4 100644 --- a/src/MSBuild/Resources/xlf/Strings.it.xlf +++ b/src/MSBuild/Resources/xlf/Strings.it.xlf @@ -315,7 +315,7 @@ Copyright (C) Microsoft Corporation. Tutti i diritti sono riservati. @<file> Inserisce le impostazioni della riga di comando da un file di testo. Per specificare più file di risposta, specificare ciascun file separatamente. - + Qualsiasi file di risposta denominato "msbuild.rsp" viene usato automaticamente dai percorsi seguenti: (1) la directory di msbuild.exe @@ -815,6 +815,7 @@ Copyright (C) Microsoft Corporation. Tutti i diritti sono riservati. file viene assegnato il nome "MSBuild<idnodo>.log". Il percorso dei file e altri parametri di fileLogger possono essere specificati aggiungendo l'opzione + "-fileLoggerParameters". Se il nome di un file di log viene impostato con l'opzione fileLoggerParameters, il logger distribuito userà il nome @@ -1685,7 +1686,8 @@ Copyright (C) Microsoft Corporation. Tutti i diritti sono riservati. file is '.md', the result is generated in markdown format. Otherwise, a tab separated file is produced. - -profileEvaluation:<file> Esegue la profilatura della valutazione di MSBuild e scrive + -profileEvaluation:<file> +Esegue la profilatura della valutazione di MSBuild e scrive il risultato nel file specificato. Se l'estensione del file specificato è '.md', il risultato viene generato in formato Markdown. In caso contrario, viene prodotto un file diff --git a/src/MSBuild/Resources/xlf/Strings.pl.xlf b/src/MSBuild/Resources/xlf/Strings.pl.xlf index d0e360c5c80..548f0673c44 100644 --- a/src/MSBuild/Resources/xlf/Strings.pl.xlf +++ b/src/MSBuild/Resources/xlf/Strings.pl.xlf @@ -412,7 +412,7 @@ Copyright (C) Microsoft Corporation. Wszelkie prawa zastrzeżone. [<klasa rejestratora>,]<zestaw rejestratora> [;<parametry rejestratora>] Składnia elementu <klasa rejestratora>: - <częściowa lub pełna przestrzeń nazw>.] + [<częściowa lub pełna przestrzeń nazw>.] <nazwa klasy rejestratora> Składnia elementu <zestaw rejestratora>: {<nazwa zestawu>[,<strong name>] | <plik zestawu>} @@ -807,6 +807,7 @@ Copyright (C) Microsoft Corporation. Wszelkie prawa zastrzeżone. Domyślnie pliki mają nazwę „MSBuild<identyfikator węzła>.log”. Lokalizację plików i inne parametry rejestratora plików można określić + przez dodanie przełącznika „-fileLoggerParameters”. Jeśli nazwa pliku zostanie ustawiona za pomocą przełącznika @@ -1641,7 +1642,7 @@ dzienników tekstowych i wykorzystać w innych narzędziach przywrócenia pakietów przed ich skompilowaniem. Podanie parametru -restore jest równoznaczne z podaniem parametru -restore:True. Za pomocą tego parametru można przesłonić wartość pochodzącą - z pliku odpowiedzi. + z pliku odpowiedzi. (Krótka forma: -r) diff --git a/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf b/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf index fc828bbf856..e41b484c1a6 100644 --- a/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf +++ b/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf @@ -1623,9 +1623,12 @@ isoladamente. -restore[:True|False] Executa um destino chamado Restore antes de compilar - outros destinos e garante o build desses   destinos usando uma lógica de build restaurada. Isso é útil quando sua árvore de -projeto precisar      que pacotes sejam restaurados antes de serem compilados.          Especificar -restore é o mesmo que - especificar -restore:True. Use o parâmetro para + outros destinos e garante o build desses + destinos usando uma lógica de build restaurada. + Isso é útil quando sua árvore de projeto precisar + que pacotes sejam restaurados antes de serem compilados. + Especificar -restore é o mesmo que + -restore:True. Use o parâmetro para substituir um valor originado de um arquivo de resposta. (Forma abreviada: -r) diff --git a/src/MSBuild/Resources/xlf/Strings.ru.xlf b/src/MSBuild/Resources/xlf/Strings.ru.xlf index 58372693c4e..4ef8b52a05a 100644 --- a/src/MSBuild/Resources/xlf/Strings.ru.xlf +++ b/src/MSBuild/Resources/xlf/Strings.ru.xlf @@ -366,7 +366,7 @@ Copyright (C) Microsoft Corporation. All rights reserved. -property:WarningLevel=2;OutDir=bin\Debug\ -property:<n>=<v> Установите или переопределите свойства уровня проекта. <n> является - именем свойства, а <v> — его значением. Используйте + именем свойства, а <v> — его значением. Используйте точку с запятой или запятую, чтобы разделить несколько свойств, или укажите каждое свойство отдельно. (Краткая форма: -p) Пример: @@ -572,12 +572,11 @@ Copyright (C) Microsoft Corporation. All rights reserved. MSBuild will use up to the number of processors on the computer. (Short form: -m[:n]) - -maxCpuCount[:n] Указывает максимальное количество - параллельных процессов сборки. Если ключ - не используется, применяется значение по умолчанию 1. - Если ключ используется без значения, MSBuild - будет использовать то количество процессоров, - которое установлено на компьютере. (Краткая форма: -m[:n]) + -maxCpuCount[:n] Указывает максимальное количество параллельных + процессов сборки. Если ключ не используется, применяется значение + по умолчанию 1. Если ключ используется без значения, + MSBuild будет использовать то количество процессоров, которое установлено на + компьютере. (Краткая форма: -m[:n]) LOCALIZATION: "maxCpuCount" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.tr.xlf b/src/MSBuild/Resources/xlf/Strings.tr.xlf index a9f98271477..8bed4f53eff 100644 --- a/src/MSBuild/Resources/xlf/Strings.tr.xlf +++ b/src/MSBuild/Resources/xlf/Strings.tr.xlf @@ -506,7 +506,7 @@ Telif Hakkı (C) Microsoft Corporation. Tüm hakları saklıdır. devre dışı bırak. EnableMPLogging--Çok işlemci olmayan modda çalışırken bile çok işlemcili günlük stilini etkinleştir. Bu - günlük stili varsayılan olarak açıktır. + günlük stili varsayılan olarak açıktır. ForceConsoleColor--Konsol desteklemese bile ANSI konsol renklerini kullan Verbosity--Bu günlükçü için /verbosity ayarını @@ -789,7 +789,7 @@ Telif Hakkı (C) Microsoft Corporation. Tüm hakları saklıdır. template and append the node id to this fileName to create a log file for each node. - -distributedFileLogger + -distributedFileLogger Derleme çıkışını, her MSBuild düğümü için bir günlük dosyası olmak üzere birden çok günlük dosyasına kaydeder. Bu dosyaların ilk konumu geçerli dizindir. Dosyaların @@ -1631,7 +1631,7 @@ Telif Hakkı (C) Microsoft Corporation. Tüm hakları saklıdır. Bu, proje ağacınızın paketlerin oluşturulabilmesi için önce geri yüklenmesini gerektirdiği durumlarda yararlıdır. -restore değerinin belirtilmesi, -restore:True değerinin - belirtilmesiyle aynıdır. Yanıt dosyasından gelen bir değeri + belirtilmesiyle aynıdır. Yanıt dosyasından gelen bir değeri geçersiz kılmak için parametreyi kullanın. (Kısa biçim: -r) diff --git a/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf b/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf index 64d2ad270db..011389fa1a6 100644 --- a/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf @@ -800,7 +800,7 @@ Copyright (C) Microsoft Corporation. All rights reserved. 开关设置的，分布式记录器将使用 fileName 作为 模板并将节点 ID 附加到此 fileName 以便为每个节点创建一个日志文件。 - + LOCALIZATION: The following should not be localized: 1) "MSBuild", "MSBuild.exe" and "MSBuild.rsp" @@ -868,7 +868,7 @@ Copyright (C) Microsoft Corporation. All rights reserved. -flp:Summary;Verbosity=minimal;LogFile=msbuild.sum -flp1:warningsonly;logfile=msbuild.wrn -flp2:errorsonly;logfile=msbuild.err - + LOCALIZATION: The following should not be localized: 1) "MSBuild", "MSBuild.exe" and "MSBuild.rsp" diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf index f30189bfe95..ca72839e962 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf @@ -2647,7 +2647,7 @@ MSB3692: Unable to create Xaml task. The <UsingTask> does not contain a <Task> definition. - MSB3692: 无法创建 Xaml 任务。 <UsingTask> 未包含 <Task> 定义。 + MSB3692: 无法创建 Xaml 任务。 <UsingTask> 未包含 <Task> 定义。 {StrBegin="MSB3692: "} @@ -3242,7 +3242,7 @@ (No message specified) - （未指定任何消息） + (未指定任何消息) From 0538acc04cc2c953f220f9ec3f3764db1769defe Mon Sep 17 00:00:00 2001 From: Ben Villalobos <4691428+BenVillalobos@users.noreply.github.com> Date: Fri, 16 Jul 2021 10:28:23 -0700 Subject: [PATCH 11/11] 16.11 Final Branding (#6656) Co-authored-by: Rainer Sigwald --- documentation/Changelog.md | 16 +++++++++++++++- eng/Versions.props | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/documentation/Changelog.md b/documentation/Changelog.md index ba4b17b5963..4a9dbaf4710 100644 --- a/documentation/Changelog.md +++ b/documentation/Changelog.md @@ -10,8 +10,9 @@ This version of MSBuild will ship with Visual Studio 2019 version 16.11.0 and .N #### Added -* Additional properties documented and available for completion in Visual Studio (#6500). +* Additional properties documented and available for completion in Visual Studio (#6500, #6530). * The `SignFile` task is now available in MSBuild on .NET 5.0 (#6509). Thanks, @Zastai! +* New version properties `MSBuildFileVersion` (4-part, matches file version) and `MSBuildSemanticVersion` (matches package versions) are now available for use (#6534). #### Changed * When using the experimental cache API, schedule proxy builds to the in-proc node for performance (#6386). @@ -24,15 +25,28 @@ This version of MSBuild will ship with Visual Studio 2019 version 16.11.0 and .N * Added locking to avoid race conditions in `BuildManager` (#6412). * Allow `ResolveAssemblyReferences` precomputed cache files to be in read-only locations (#6393). * 64-bit `al.exe` is used when targeting 64-bit architectures (for real this time) (#6484). +* Builds with `ProduceOnlyReferenceAssembly` no longer expect debug symbols to be produced (#6511). Thanks, @Zastai! #### Infrastructure * Use a packaged C# compiler to avoid changes in reference assembly generation caused by compiler changes (#6431). * Use more resilient test-result upload patterns (#6489). * Conditional compilation for .NET Core within our repo now includes new .NET 5.0+ runtimes (#6538). +* Switched to OneLocBuild for localization PRs (#6561). +* Moved to latest Ubuntu image for PR test legs (#6573). #### Documentation +## MSBuild 16.10.2 + +This version of MSBuild shipped with Visual Studio 2019 version 16.10.2 and will ship with .NET SDK 5.0.302. + +#### Fixed + +* Fixed a regression in the `MakeRelative` property function that dropped trailing slashes (#6513). Thanks, @dsparkplug and @pmisik! +* Fixed a regression in glob matching where files without extensions were erroneously not matched (#6531). +* Fixed a change in logging that caused crashes in Azure DevOps loggers (#6520). + ## MSBuild 16.10.1 This version of MSBuild shipped with Visual Studio 2019 version 16.10.1 and .NET SDK 5.0.301. diff --git a/eng/Versions.props b/eng/Versions.props index 58e4de7b2d9..af8c04c9ee0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,7 +2,7 @@ - 16.11.0 + 16.11.0release 15.1.0.0 preview true