diff --git a/documentation/project-docs/telemetry.md b/documentation/project-docs/telemetry.md index e7ec6a03e46a..e559f60bb25b 100644 --- a/documentation/project-docs/telemetry.md +++ b/documentation/project-docs/telemetry.md @@ -61,7 +61,7 @@ Every telemetry event automatically includes these common properties: | **Telemetry Profile** | Custom telemetry profile (if set via env var) | Custom value or null | | **Docker Container** | Whether running in Docker container | `True` or `False` | | **CI** | Whether running in CI environment | `True` or `False` | -| **LLM** | Detected LLM/assistant environment identifiers (comma-separated) | `claude` or `cursor` | +| **LLM** | Detected LLM/assistant environment identifiers (comma-separated) | `claude`, `cursor`, `gemini`, `copilot`, `generic_agent` | | **Current Path Hash** | SHA256 hash of current directory path | Hashed value | | **Machine ID** | SHA256 hash of machine MAC address (or GUID if unavailable) | Hashed value | | **Machine ID Old** | Legacy machine ID for compatibility | Hashed value | diff --git a/src/Cli/Microsoft.DotNet.Cli.Utils/Constants.cs b/src/Cli/Microsoft.DotNet.Cli.Utils/Constants.cs index 69b532afb0bb..0c5aef161367 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Utils/Constants.cs +++ b/src/Cli/Microsoft.DotNet.Cli.Utils/Constants.cs @@ -35,6 +35,14 @@ public static class Constants public const string Identity = nameof(Identity); public const string FullPath = nameof(FullPath); + // MSBuild CLI flags + + /// + /// Disables the live-updating node display in the terminal logger, which is useful for LLM/agentic environments. + /// + public const string TerminalLogger_DisableNodeDisplay = "-tlp:DISABLENODEDISPLAY"; + + public static readonly string ProjectArgumentName = ""; public static readonly string SolutionArgumentName = ""; public static readonly string ToolPackageArgumentName = ""; diff --git a/src/Cli/Microsoft.DotNet.Cli.Utils/MSBuildForwardingAppWithoutLogging.cs b/src/Cli/Microsoft.DotNet.Cli.Utils/MSBuildForwardingAppWithoutLogging.cs index 97e7abd086c3..e0dca709ac3c 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Utils/MSBuildForwardingAppWithoutLogging.cs +++ b/src/Cli/Microsoft.DotNet.Cli.Utils/MSBuildForwardingAppWithoutLogging.cs @@ -56,7 +56,6 @@ public MSBuildForwardingAppWithoutLogging(MSBuildArgs msbuildArgs, string? msbui msbuildArgs.OtherMSBuildArgs.Add("-nologo"); } string? tlpDefault = TerminalLoggerDefault; - // new for .NET 9 - default TL to auto (aka enable in non-CI scenarios) if (string.IsNullOrWhiteSpace(tlpDefault)) { tlpDefault = "auto"; diff --git a/src/Cli/dotnet/Commands/MSBuild/MSBuildForwardingApp.cs b/src/Cli/dotnet/Commands/MSBuild/MSBuildForwardingApp.cs index 67c2a47f1051..3055de0883f5 100644 --- a/src/Cli/dotnet/Commands/MSBuild/MSBuildForwardingApp.cs +++ b/src/Cli/dotnet/Commands/MSBuild/MSBuildForwardingApp.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Reflection; +using Microsoft.DotNet.Cli.Commands.Run; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.Cli.Utils.Extensions; @@ -14,6 +15,9 @@ public class MSBuildForwardingApp : CommandBase private readonly MSBuildForwardingAppWithoutLogging _forwardingAppWithoutLogging; + /// + /// Adds the CLI's telemetry logger to the MSBuild arguments if telemetry is enabled. + /// private static MSBuildArgs ConcatTelemetryLogger(MSBuildArgs msbuildArgs) { if (Telemetry.Telemetry.CurrentSessionId != null) @@ -45,8 +49,9 @@ public MSBuildForwardingApp(IEnumerable rawMSBuildArgs, string? msbuildP public MSBuildForwardingApp(MSBuildArgs msBuildArgs, string? msbuildPath = null, bool includeLogo = false) { + var modifiedMSBuildArgs = CommonRunHelpers.AdjustMSBuildForLLMs(ConcatTelemetryLogger(msBuildArgs)); _forwardingAppWithoutLogging = new MSBuildForwardingAppWithoutLogging( - ConcatTelemetryLogger(msBuildArgs), + modifiedMSBuildArgs, msbuildPath: msbuildPath, includeLogo: includeLogo); diff --git a/src/Cli/dotnet/Commands/Run/CommonRunHelpers.cs b/src/Cli/dotnet/Commands/Run/CommonRunHelpers.cs index eed2437e281c..64d2b8a1075a 100644 --- a/src/Cli/dotnet/Commands/Run/CommonRunHelpers.cs +++ b/src/Cli/dotnet/Commands/Run/CommonRunHelpers.cs @@ -26,4 +26,29 @@ public static string GetPropertiesLaunchSettingsPath(string directoryPath, strin public static string GetFlatLaunchSettingsPath(string directoryPath, string projectNameWithoutExtension) => Path.Join(directoryPath, $"{projectNameWithoutExtension}.run.json"); + + + /// + /// Applies adjustments to MSBuild arguments to better suit LLM/agentic environments, if such an environment is detected. + /// + public static MSBuildArgs AdjustMSBuildForLLMs(MSBuildArgs msbuildArgs) + { + if (new Telemetry.LLMEnvironmentDetectorForTelemetry().IsLLMEnvironment()) + { + // disable the live-update display of the TerminalLogger, which wastes tokens + return msbuildArgs.CloneWithAdditionalArgs(Constants.TerminalLogger_DisableNodeDisplay); + } + else + { + return msbuildArgs; + } + } + + /// + /// Creates a TerminalLogger or ConsoleLogger based on the provided MSBuild arguments. + /// If the environment is detected to be an LLM environment, the logger is adjusted to + /// better suit that environment. + /// + public static Microsoft.Build.Framework.ILogger GetConsoleLogger(MSBuildArgs args) => + Microsoft.Build.Logging.TerminalLogger.CreateTerminalOrConsoleLogger([.. AdjustMSBuildForLLMs(args).OtherMSBuildArgs]); } diff --git a/src/Cli/dotnet/Commands/Run/RunCommand.cs b/src/Cli/dotnet/Commands/Run/RunCommand.cs index 183569417ae2..635566bf07b4 100644 --- a/src/Cli/dotnet/Commands/Run/RunCommand.cs +++ b/src/Cli/dotnet/Commands/Run/RunCommand.cs @@ -483,7 +483,9 @@ static ICommand CreateCommandForCscBuiltProgram(string entryPointFileFullPath, s static void InvokeRunArgumentsTarget(ProjectInstance project, bool noBuild, FacadeLogger? binaryLogger, MSBuildArgs buildArgs) { List loggersForBuild = [ - TerminalLogger.CreateTerminalOrConsoleLogger([$"--verbosity:{LoggerVerbosity.Quiet.ToString().ToLowerInvariant()}", ..buildArgs.OtherMSBuildArgs]) + CommonRunHelpers.GetConsoleLogger( + buildArgs.CloneWithExplicitArgs([$"--verbosity:{LoggerVerbosity.Quiet.ToString().ToLowerInvariant()}", ..buildArgs.OtherMSBuildArgs]) + ) ]; if (binaryLogger is not null) { diff --git a/src/Cli/dotnet/Commands/Run/VirtualProjectBuildingCommand.cs b/src/Cli/dotnet/Commands/Run/VirtualProjectBuildingCommand.cs index 17f5ce1b226e..794b7f77974e 100644 --- a/src/Cli/dotnet/Commands/Run/VirtualProjectBuildingCommand.cs +++ b/src/Cli/dotnet/Commands/Run/VirtualProjectBuildingCommand.cs @@ -190,7 +190,7 @@ public override int Execute() var verbosity = MSBuildArgs.Verbosity ?? MSBuildForwardingAppWithoutLogging.DefaultVerbosity; var consoleLogger = minimizeStdOut ? new SimpleErrorLogger() - : TerminalLogger.CreateTerminalOrConsoleLogger([$"--verbosity:{verbosity}", .. MSBuildArgs.OtherMSBuildArgs]); + : CommonRunHelpers.GetConsoleLogger(MSBuildArgs.CloneWithExplicitArgs([$"--verbosity:{verbosity}", .. MSBuildArgs.OtherMSBuildArgs])); var binaryLogger = GetBinaryLogger(MSBuildArgs.OtherMSBuildArgs); CacheInfo? cache = null; diff --git a/src/Cli/dotnet/Telemetry/EnvironmentDetectionRule.cs b/src/Cli/dotnet/Telemetry/EnvironmentDetectionRule.cs index 5cd73f53abb8..ebdf6321ddd7 100644 --- a/src/Cli/dotnet/Telemetry/EnvironmentDetectionRule.cs +++ b/src/Cli/dotnet/Telemetry/EnvironmentDetectionRule.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.DotNet.Cli.Utils; namespace Microsoft.DotNet.Cli.Telemetry; @@ -33,8 +34,7 @@ public BooleanEnvironmentRule(params string[] variables) public override bool IsMatch() { - return _variables.Any(variable => - bool.TryParse(Environment.GetEnvironmentVariable(variable), out bool value) && value); + return _variables.Any(variable => Env.GetEnvironmentVariableAsBool(variable)); } } @@ -81,12 +81,12 @@ public override bool IsMatch() /// The type of the result value. internal class EnvironmentDetectionRuleWithResult where T : class { - private readonly string[] _variables; + private readonly EnvironmentDetectionRule _rule; private readonly T _result; - public EnvironmentDetectionRuleWithResult(T result, params string[] variables) + public EnvironmentDetectionRuleWithResult(T result, EnvironmentDetectionRule rule) { - _variables = variables ?? throw new ArgumentNullException(nameof(variables)); + _rule = rule ?? throw new ArgumentNullException(nameof(rule)); _result = result ?? throw new ArgumentNullException(nameof(result)); } @@ -96,8 +96,8 @@ public EnvironmentDetectionRuleWithResult(T result, params string[] variables) /// The result value if the rule matches; otherwise, null. public T? GetResult() { - return _variables.Any(variable => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable))) - ? _result + return _rule.IsMatch() + ? _result : null; } -} \ No newline at end of file +} diff --git a/src/Cli/dotnet/Telemetry/ILLMEnvironmentDetector.cs b/src/Cli/dotnet/Telemetry/ILLMEnvironmentDetector.cs index fe599569aa6c..e2ee21591567 100644 --- a/src/Cli/dotnet/Telemetry/ILLMEnvironmentDetector.cs +++ b/src/Cli/dotnet/Telemetry/ILLMEnvironmentDetector.cs @@ -5,5 +5,13 @@ namespace Microsoft.DotNet.Cli.Telemetry; internal interface ILLMEnvironmentDetector { + /// + /// Checks the current environment for known indicators of LLM usage and returns a string identifying the LLM environment if detected. + /// string? GetLLMEnvironment(); -} \ No newline at end of file + + /// + /// Returns true if the current environment is detected to be an LLM/agentic environment, false otherwise. + /// + bool IsLLMEnvironment(); +} diff --git a/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs b/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs index 16d13a6879e7..b37f9b5d0830 100644 --- a/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs +++ b/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs @@ -1,23 +1,30 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Linq; - namespace Microsoft.DotNet.Cli.Telemetry; internal class LLMEnvironmentDetectorForTelemetry : ILLMEnvironmentDetector { private static readonly EnvironmentDetectionRuleWithResult[] _detectionRules = [ // Claude Code - new EnvironmentDetectionRuleWithResult("claude", "CLAUDECODE"), + new EnvironmentDetectionRuleWithResult("claude", new AnyPresentEnvironmentRule("CLAUDECODE")), // Cursor AI - new EnvironmentDetectionRuleWithResult("cursor", "CURSOR_EDITOR") + new EnvironmentDetectionRuleWithResult("cursor", new AnyPresentEnvironmentRule("CURSOR_EDITOR")), + // Gemini + new EnvironmentDetectionRuleWithResult("gemini", new BooleanEnvironmentRule("GEMINI_CLI")), + // GitHub Copilot + new EnvironmentDetectionRuleWithResult("copilot", new BooleanEnvironmentRule("GITHUB_COPILOT_CLI_MODE")), + // (proposed) generic flag for Agentic usage + new EnvironmentDetectionRuleWithResult("generic_agent", new BooleanEnvironmentRule("AGENT_CLI")), ]; + /// public string? GetLLMEnvironment() { var results = _detectionRules.Select(r => r.GetResult()).Where(r => r != null).ToArray(); return results.Length > 0 ? string.Join(", ", results) : null; } -} \ No newline at end of file + + /// + public bool IsLLMEnvironment() => !string.IsNullOrEmpty(GetLLMEnvironment()); +} diff --git a/test/dotnet.Tests/CommandTests/MSBuild/GivenDotnetBuildInvocation.cs b/test/dotnet.Tests/CommandTests/MSBuild/GivenDotnetBuildInvocation.cs index 08c5ee7071cd..8b6c7de5b074 100644 --- a/test/dotnet.Tests/CommandTests/MSBuild/GivenDotnetBuildInvocation.cs +++ b/test/dotnet.Tests/CommandTests/MSBuild/GivenDotnetBuildInvocation.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.DotNet.Cli.Commands.Restore; +using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.Tests; using BuildCommand = Microsoft.DotNet.Cli.Commands.Build.BuildCommand; namespace Microsoft.DotNet.Cli.MSBuild.Tests @@ -10,8 +12,8 @@ namespace Microsoft.DotNet.Cli.MSBuild.Tests public class GivenDotnetBuildInvocation : IClassFixture { string[] ExpectedPrefix = ["-maxcpucount", "--verbosity:m", "-tlp:default=auto", "-nologo"]; - public static string[] RestoreExpectedPrefixForImplicitRestore = [..RestoringCommand.RestoreOptimizationProperties.Select(kvp => $"--restoreProperty:{kvp.Key}={kvp.Value}")]; - public static string[] RestoreExpectedPrefixForSeparateRestore = [..RestoringCommand.RestoreOptimizationProperties.Select(kvp => $"--property:{kvp.Key}={kvp.Value}")]; + public static string[] RestoreExpectedPrefixForImplicitRestore = [.. RestoringCommand.RestoreOptimizationProperties.Select(kvp => $"--restoreProperty:{kvp.Key}={kvp.Value}")]; + public static string[] RestoreExpectedPrefixForSeparateRestore = [.. RestoringCommand.RestoreOptimizationProperties.Select(kvp => $"--property:{kvp.Key}={kvp.Value}")]; const string NugetInteractiveProperty = "--property:NuGetInteractive=false"; @@ -118,5 +120,47 @@ public void MsbuildInvocationIsCorrectForSeparateRestore( .BeEquivalentTo([.. ExpectedPrefix, "-consoleloggerparameters:Summary", NugetInteractiveProperty, .. expectedAdditionalArgs]); }); } + + [Theory] + [MemberData(memberName: nameof(TelemetryCommonPropertiesTests.LLMTelemetryTestCases), MemberType =typeof(TelemetryCommonPropertiesTests))] + public void WhenLLMIsDetectedTLLiveUpdateIsDisabled(Dictionary? llmEnvVarsToSet, string? expectedLLMName) + { + CommandDirectoryContext.PerformActionWithBasePath(WorkingDirectory, () => + { + try + { + // Set environment variables to simulate LLM environment + if (llmEnvVarsToSet is not null) + { + foreach (var (key, value) in llmEnvVarsToSet) + { + Environment.SetEnvironmentVariable(key, value); + } + } + + var command = (RestoringCommand)BuildCommand.FromArgs([]); + + if (expectedLLMName is not null) + { + command.GetArgumentTokensToMSBuild().Should().Contain(Constants.TerminalLogger_DisableNodeDisplay); + } + else + { + command.GetArgumentTokensToMSBuild().Should().NotContain(Constants.TerminalLogger_DisableNodeDisplay); + } + } + finally + { + // Clear the environment variables after the test + if (llmEnvVarsToSet is not null) + { + foreach (var (key, value) in llmEnvVarsToSet) + { + Environment.SetEnvironmentVariable(key, null); + } + } + } + }); + } } } diff --git a/test/dotnet.Tests/TelemetryCommonPropertiesTests.cs b/test/dotnet.Tests/TelemetryCommonPropertiesTests.cs index a77f0bf81765..4e28b92479d7 100644 --- a/test/dotnet.Tests/TelemetryCommonPropertiesTests.cs +++ b/test/dotnet.Tests/TelemetryCommonPropertiesTests.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - using Microsoft.DotNet.Cli.Telemetry; using Microsoft.DotNet.Configurer; @@ -193,29 +191,34 @@ public void CanDetectCIStatusForEnvVars(Dictionary envVars, bool [Theory] [MemberData(nameof(LLMTelemetryTestCases))] - public void CanDetectLLMStatusForEnvVars(Dictionary envVars, string expected) + public void CanDetectLLMStatusForEnvVars(Dictionary? envVars, string? expected) { try { - foreach (var (key, value) in envVars) - { - Environment.SetEnvironmentVariable(key, value); + if (envVars is not null){ + foreach (var (key, value) in envVars) + { + Environment.SetEnvironmentVariable(key, value); + } } new LLMEnvironmentDetectorForTelemetry().GetLLMEnvironment().Should().Be(expected); } finally { - foreach (var (key, value) in envVars) + if (envVars is not null) { - Environment.SetEnvironmentVariable(key, null); + foreach (var (key, value) in envVars) + { + Environment.SetEnvironmentVariable(key, null); + } } } } - + [Theory] [InlineData("dummySessionId")] [InlineData(null)] - public void TelemetryCommonPropertiesShouldContainSessionId(string sessionId) + public void TelemetryCommonPropertiesShouldContainSessionId(string? sessionId) { var unitUnderTest = new TelemetryCommonProperties(userLevelCacheWriter: new NothingCache()); var commonProperties = unitUnderTest.GetTelemetryCommonProperties(sessionId); @@ -225,34 +228,42 @@ public void TelemetryCommonPropertiesShouldContainSessionId(string sessionId) } - public static IEnumerable LLMTelemetryTestCases => new List{ - new object[] { new Dictionary { { "CLAUDECODE", "1" } }, "claude" }, - new object[] { new Dictionary { { "CURSOR_EDITOR", "1" } }, "cursor" }, - new object[] { new Dictionary { { "CLAUDECODE", "1" }, { "CURSOR_EDITOR", "1" } }, "claude, cursor" }, - new object[] { new Dictionary(), null }, + public static TheoryData?, string?> LLMTelemetryTestCases => new() + { + { new Dictionary { {"CLAUDECODE", "1" } }, "claude" }, + { new Dictionary { { "CURSOR_EDITOR", "1" } }, "cursor" }, + { new Dictionary { { "GEMINI_CLI", "true" } }, "gemini" }, + { new Dictionary { { "GITHUB_COPILOT_CLI_MODE", "true" } }, "copilot" }, + { new Dictionary { { "AGENT_CLI", "true" } }, "generic_agent" }, + { new Dictionary { { "CLAUDECODE", "1" }, { "CURSOR_EDITOR", "1" } }, "claude, cursor" }, + { new Dictionary { { "GEMINI_CLI", "true" }, { "GITHUB_COPILOT_CLI_MODE", "true" } }, "gemini, copilot" }, + { new Dictionary { { "CLAUDECODE", "1" }, { "GEMINI_CLI", "true" }, { "AGENT_CLI", "true" } }, "claude, gemini, generic_agent" }, + { new Dictionary { { "CLAUDECODE", "1" }, { "CURSOR_EDITOR", "1" }, { "GEMINI_CLI", "true" }, { "GITHUB_COPILOT_CLI_MODE", "true" }, { "AGENT_CLI", "true" } }, "claude, cursor, gemini, copilot, generic_agent" }, + { new Dictionary { { "GEMINI_CLI", "false" } }, null }, + { new Dictionary { { "GITHUB_COPILOT_CLI_MODE", "false" } }, null }, + { new Dictionary { { "AGENT_CLI", "false" } }, null }, + { new Dictionary(), null }, }; - public static IEnumerable CITelemetryTestCases => new List{ - new object[] { new Dictionary { { "TF_BUILD", "true" } }, true }, - new object[] { new Dictionary { { "GITHUB_ACTIONS", "true" } }, true }, - new object[] { new Dictionary { { "APPVEYOR", "true"} }, true }, - new object[] { new Dictionary { { "CI", "true"} }, true }, - new object[] { new Dictionary { { "TRAVIS", "true"} }, true }, - new object[] { new Dictionary { { "CIRCLECI", "true"} }, true }, - - new object[] { new Dictionary { { "CODEBUILD_BUILD_ID", "hi" }, { "AWS_REGION", "hi" } }, true }, - new object[] { new Dictionary { { "CODEBUILD_BUILD_ID", "hi" } }, false }, - new object[] { new Dictionary { { "BUILD_ID", "hi" }, { "BUILD_URL", "hi" } }, true }, - new object[] { new Dictionary { { "BUILD_ID", "hi" } }, false }, - new object[] { new Dictionary { { "BUILD_ID", "hi" }, { "PROJECT_ID", "hi" } }, true }, - new object[] { new Dictionary { { "BUILD_ID", "hi" } }, false }, - - new object[] { new Dictionary { { "TEAMCITY_VERSION", "hi" } }, true }, - new object[] { new Dictionary { { "TEAMCITY_VERSION", "" } }, false }, - new object[] { new Dictionary { { "JB_SPACE_API_URL", "hi" } }, true }, - new object[] { new Dictionary { { "JB_SPACE_API_URL", "" } }, false }, - - new object[] { new Dictionary { { "SomethingElse", "hi" } }, false }, + public static TheoryData, bool> CITelemetryTestCases => new() + { + { new Dictionary { { "TF_BUILD", "true" } }, true }, + { new Dictionary { { "GITHUB_ACTIONS", "true" } }, true }, + { new Dictionary { { "APPVEYOR", "true"} }, true }, + { new Dictionary { { "CI", "true"} }, true }, + { new Dictionary { { "TRAVIS", "true"} }, true }, + { new Dictionary { { "CIRCLECI", "true"} }, true }, +{ new Dictionary { { "CODEBUILD_BUILD_ID", "hi" }, { "AWS_REGION", "hi" } }, true }, + { new Dictionary { { "CODEBUILD_BUILD_ID", "hi" } }, false }, + { new Dictionary { { "BUILD_ID", "hi" }, { "BUILD_URL", "hi" } }, true }, + { new Dictionary { { "BUILD_ID", "hi" } }, false }, + { new Dictionary { { "BUILD_ID", "hi" }, { "PROJECT_ID", "hi" } }, true }, + { new Dictionary { { "BUILD_ID", "hi" } }, false }, +{ new Dictionary { { "TEAMCITY_VERSION", "hi" } }, true }, + { new Dictionary { { "TEAMCITY_VERSION", "" } }, false }, + { new Dictionary { { "JB_SPACE_API_URL", "hi" } }, true }, + { new Dictionary { { "JB_SPACE_API_URL", "" } }, false }, +{ new Dictionary { { "SomethingElse", "hi" } }, false }, }; private class NothingCache : IUserLevelCacheWriter