Add telemetry to categorize build failure reasons

documentation/wiki/CollectedTelemetry.md

@@ -100,6 +100,14 @@ Expressed and collected via [BuildTelemetry type](https://github.com/dotnet/msbu
 | >= 9.0.100   | Indication of enablement of BuildCheck feature. |
 | >= 9.0.100   | Indication of Smart App Control being in evaluation mode on machine executing the build. |
 | >= 10.0.100  | Indication if the build was run in multithreaded mode. |
+| >= 10.0.200  | Primary failure category when BuildSuccess = false (one of: "Compiler", "MSBuildEngine", "Tasks", "SDK", "NuGet", "BuildCheck", "Other"). |
+| >= 10.0.200  | Count of compiler errors encountered during the build. |
+| >= 10.0.200  | Count of MSBuild engine errors encountered during the build. |
+| >= 10.0.200  | Count of task errors encountered during the build. |
+| >= 10.0.200  | Count of SDK errors encountered during the build. |
+| >= 10.0.200  | Count of NuGet errors encountered during the build. |
+| >= 10.0.200  | Count of BuildCheck errors encountered during the build. |
+| >= 10.0.200  | Count of other errors encountered during the build. |
 
 ### Project Build
 

src/Build.UnitTests/BackEnd/BuildTelemetryErrorCategorization_Tests.cs

@@ -0,0 +1,215 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Build.BackEnd.Logging;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Framework.Telemetry;
+using Microsoft.Build.Shared;
+using Shouldly;
+using Xunit;
+
+#nullable disable
+
+namespace Microsoft.Build.UnitTests.BackEnd;
+
+public class BuildTelemetryErrorCategorization_Tests
+{
+    [Theory]
+    [InlineData("CS0103", null, "Compiler")]
+    [InlineData("CS1002", "CS", "Compiler")]
+    [InlineData("VBC30451", "VBC", "Compiler")]
+    [InlineData("FS0039", null, "Compiler")]
+    [InlineData("MSB4018", null, "MSBuildEngine")]
+    [InlineData("MSB4236", null, "SDKResolvers")]
+    [InlineData("MSB3026", null, "Tasks")]
+    [InlineData("NETSDK1045", null, "NETSDK")]
+    [InlineData("NU1101", null, "NuGet")]
+    [InlineData("BC0001", null, "BuildCheck")]
+    [InlineData("CUSTOM001", null, "Other")]
+    [InlineData(null, null, "Other")]
+    [InlineData("", null, "Other")]
+    public void ErrorCategorizationWorksCorrectly(string errorCode, string subcategory, string expectedCategory)
+    {
+        // Create a LoggingService
+        var loggingService = LoggingService.CreateLoggingService(LoggerMode.Synchronous, 1);
+        loggingService.OnlyLogCriticalEvents = false;
+
+        try
+        {
+            // Log an error with the specified code
+            var errorEvent = new BuildErrorEventArgs(
+                subcategory,
+                errorCode,
+                "file.cs",
+                1,
+                1,
+                0,
+                0,
+                "Test error message",
+                "helpKeyword",
+                "sender");
+
+            loggingService.LogBuildEvent(errorEvent);
+
+            // Populate telemetry
+            var buildTelemetry = new BuildTelemetry();
+            loggingService.PopulateBuildTelemetryWithErrors(buildTelemetry);
+
+            // Verify the category is set correctly
+            buildTelemetry.FailureCategory.ShouldBe(expectedCategory);
+
+            // Verify the appropriate count is incremented
+            switch (expectedCategory)
+            {
+                case "Compiler":
+                    buildTelemetry.ErrorCounts.Compiler.ShouldBe(1);
+                    break;
+                case "MSBuildEngine":
+                    buildTelemetry.ErrorCounts.MsBuildEngine.ShouldBe(1);
+                    break;
+                case "Tasks":
+                    buildTelemetry.ErrorCounts.Task.ShouldBe(1);
+                    break;
+                case "SDKResolvers":
+                    buildTelemetry.ErrorCounts.SdkResolvers.ShouldBe(1);
+                    break;
+                case "NETSDK":
+                    buildTelemetry.ErrorCounts.NetSdk.ShouldBe(1);
+                    break;
+                case "NuGet":
+                    buildTelemetry.ErrorCounts.NuGet.ShouldBe(1);
+                    break;
+                case "BuildCheck":
+                    buildTelemetry.ErrorCounts.BuildCheck.ShouldBe(1);
+                    break;
+                case "Other":
+                    buildTelemetry.ErrorCounts.Other.ShouldBe(1);
+                    break;
+            }
+        }
+        finally
+        {
+            loggingService.ShutdownComponent();
+        }
+    }
+
+    [Fact]
+    public void MultipleErrorsAreCountedByCategory()
+    {
+        var loggingService = LoggingService.CreateLoggingService(LoggerMode.Synchronous, 1);
+        loggingService.OnlyLogCriticalEvents = false;
+
+        try
+        {
+            // Log multiple errors of different categories
+            var errors = new[]
+            {
+                new BuildErrorEventArgs(null, "CS0103", "file.cs", 1, 1, 0, 0, "Error 1", null, "sender"),
+                new BuildErrorEventArgs(null, "CS1002", "file.cs", 2, 1, 0, 0, "Error 2", null, "sender"),
+                new BuildErrorEventArgs(null, "MSB4018", "file.proj", 10, 5, 0, 0, "Error 3", null, "sender"),
+                new BuildErrorEventArgs(null, "MSB3026", "file.proj", 15, 3, 0, 0, "Error 4", null, "sender"),
+                new BuildErrorEventArgs(null, "NU1101", "file.proj", 20, 1, 0, 0, "Error 5", null, "sender"),
+                new BuildErrorEventArgs(null, "CUSTOM001", "file.txt", 1, 1, 0, 0, "Error 6", null, "sender"),
+            };
+
+            foreach (var error in errors)
+            {
+                loggingService.LogBuildEvent(error);
+            }
+
+            // Populate telemetry
+            var buildTelemetry = new BuildTelemetry();
+            loggingService.PopulateBuildTelemetryWithErrors(buildTelemetry);
+
+            // Verify counts
+            buildTelemetry.ErrorCounts.Compiler.ShouldBe(2);
+            buildTelemetry.ErrorCounts.MsBuildEngine.ShouldBe(1);
+            buildTelemetry.ErrorCounts.Task.ShouldBe(1);
+            buildTelemetry.ErrorCounts.NuGet.ShouldBe(1);
+            buildTelemetry.ErrorCounts.Other.ShouldBe(1);
+
+            // Primary category should be Compiler (highest count)
+            buildTelemetry.FailureCategory.ShouldBe("Compiler");
+        }
+        finally
+        {
+            loggingService.ShutdownComponent();
+        }
+    }
+
+    [Fact]
+    public void PrimaryCategoryIsSetToHighestErrorCount()
+    {
+        var loggingService = LoggingService.CreateLoggingService(LoggerMode.Synchronous, 1);
+        loggingService.OnlyLogCriticalEvents = false;
+
+        try
+        {
+            // Log errors with Tasks having the highest count
+            var errors = new[]
+            {
+                new BuildErrorEventArgs(null, "MSB3026", "file.proj", 1, 1, 0, 0, "Task Error 1", null, "sender"),
+                new BuildErrorEventArgs(null, "MSB3027", "file.proj", 2, 1, 0, 0, "Task Error 2", null, "sender"),
+                new BuildErrorEventArgs(null, "MSB3028", "file.proj", 3, 1, 0, 0, "Task Error 3", null, "sender"),
+                new BuildErrorEventArgs(null, "CS0103", "file.cs", 4, 1, 0, 0, "Compiler Error", null, "sender"),
+            };
+
+            foreach (var error in errors)
+            {
+                loggingService.LogBuildEvent(error);
+            }
+
+            // Populate telemetry
+            var buildTelemetry = new BuildTelemetry();
+            loggingService.PopulateBuildTelemetryWithErrors(buildTelemetry);
+
+            // Primary category should be Tasks (3 errors vs 1 compiler error)
+            buildTelemetry.FailureCategory.ShouldBe("Tasks");
+            buildTelemetry.ErrorCounts.Task.ShouldBe(3);
+            buildTelemetry.ErrorCounts.Compiler.ShouldBe(1);
+        }
+        finally
+        {
+            loggingService.ShutdownComponent();
+        }
+    }
+
+    [Fact]
+    public void SubcategoryIsUsedForCompilerErrors()
+    {
+        var loggingService = LoggingService.CreateLoggingService(LoggerMode.Synchronous, 1);
+        loggingService.OnlyLogCriticalEvents = false;
+
+        try
+        {
+            // Log an error with subcategory "CS" (common for C# compiler errors)
+            var errorEvent = new BuildErrorEventArgs(
+                "CS",  // subcategory
+                "CS0103",
+                "file.cs",
+                1,
+                1,
+                0,
+                0,
+                "The name 'foo' does not exist in the current context",
+                "helpKeyword",
+                "csc");
+
+            loggingService.LogBuildEvent(errorEvent);
+
+            // Populate telemetry
+            var buildTelemetry = new BuildTelemetry();
+            loggingService.PopulateBuildTelemetryWithErrors(buildTelemetry);
+
+            // Should be categorized as Compiler based on subcategory
+            buildTelemetry.FailureCategory.ShouldBe("Compiler");
+            buildTelemetry.ErrorCounts.Compiler.ShouldBe(1);
+        }
+        finally
+        {
+            loggingService.ShutdownComponent();
+        }
+    }
+}

src/Build.UnitTests/BackEnd/KnownTelemetry_Tests.cs

@@ -7,6 +7,7 @@
 using Microsoft.Build.Framework.Telemetry;
 using Shouldly;
 using Xunit;
+using static Microsoft.Build.Framework.Telemetry.BuildInsights;
 
 namespace Microsoft.Build.UnitTests.Telemetry;
 
@@ -123,4 +124,67 @@ public void BuildTelemetryHandleNullsInRecordedTimes()
         buildTelemetry.FinishedAt = DateTime.MaxValue;
         buildTelemetry.GetProperties().ShouldBeEmpty();
     }
+
+    [Fact]
+    public void BuildTelemetryIncludesFailureCategoryProperties()
+    {
+        BuildTelemetry buildTelemetry = new BuildTelemetry();
+
+        buildTelemetry.BuildSuccess = false;
+        buildTelemetry.FailureCategory = "Compiler";
+        buildTelemetry.ErrorCounts = new ErrorCountsInfo(
+            Compiler: 5,
+            MsBuildEngine: 2,
+            Task: 1,
+            SdkResolvers: null,
+            NetSdk: null,
+            NuGet: 3,
+            BuildCheck: null,
+            Other: 1);
+
+        var properties = buildTelemetry.GetProperties();
+
+        properties["BuildSuccess"].ShouldBe("False");
+        properties["FailureCategory"].ShouldBe("Compiler");
+        properties.ContainsKey("ErrorCounts").ShouldBeTrue();
+
+        var activityProperties = buildTelemetry.GetActivityProperties();
+        activityProperties["FailureCategory"].ShouldBe("Compiler");
+        var errorCounts = activityProperties["ErrorCounts"] as ErrorCountsInfo;
+        errorCounts.ShouldNotBeNull();
+        errorCounts.Compiler.ShouldBe(5);
+        errorCounts.MsBuildEngine.ShouldBe(2);
+        errorCounts.Task.ShouldBe(1);
+        errorCounts.NuGet.ShouldBe(3);
+        errorCounts.Other.ShouldBe(1);
+        errorCounts.SdkResolvers.ShouldBeNull();
+        errorCounts.NetSdk.ShouldBeNull();
+        errorCounts.BuildCheck.ShouldBeNull();
+    }
+
+    [Fact]
+    public void BuildTelemetryActivityPropertiesIncludesFailureData()
+    {
+        BuildTelemetry buildTelemetry = new BuildTelemetry();
+
+        buildTelemetry.BuildSuccess = false;
+        buildTelemetry.FailureCategory = "Tasks";
+        buildTelemetry.ErrorCounts = new ErrorCountsInfo(
+            Compiler: null,
+            MsBuildEngine: null,
+            Task: 10,
+            SdkResolvers: null,
+            NetSdk: null,
+            NuGet: null,
+            BuildCheck: null,
+            Other: null);
+
+        var activityProperties = buildTelemetry.GetActivityProperties();
+
+        activityProperties["BuildSuccess"].ShouldBe(false);
+        activityProperties["FailureCategory"].ShouldBe("Tasks");
+        var errorCounts = activityProperties["ErrorCounts"] as ErrorCountsInfo;
+        errorCounts.ShouldNotBeNull();
+        errorCounts.Task.ShouldBe(10);
+    }
 }

src/Build.UnitTests/BackEnd/MockLoggingService.cs

@@ -677,6 +677,11 @@ public bool HasBuildSubmissionLoggedErrors(int submissionId)
             return false;
         }
 
+        public void PopulateBuildTelemetryWithErrors(Framework.Telemetry.BuildTelemetry buildTelemetry)
+        {
+            // Mock implementation does nothing
+        }
+
         public ICollection<string> GetWarningsAsErrors(BuildEventContext context)
         {
             throw new NotImplementedException();

src/Build/BackEnd/BuildManager/BuildManager.cs

@@ -1092,6 +1092,12 @@ public void EndBuild()
                             _buildTelemetry.BuildEngineDisplayVersion = ProjectCollection.DisplayVersion;
                             _buildTelemetry.BuildEngineFrameworkName = NativeMethodsShared.FrameworkName;
 
+                            // Populate error categorization data from the logging service
+                            if (!_overallBuildSuccess)
+                            {
+                                loggingService.PopulateBuildTelemetryWithErrors(_buildTelemetry);
+                            }
+
                             string? host = null;
                             if (BuildEnvironmentState.s_runningInVisualStudio)
                             {