diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 34dcba7177..3c98eeaa4f 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -32,7 +32,7 @@ 9.0.1 4.7.63 16.9.0-preview-4267359 - 16.9.0-beta.21064.1 + 16.9.0-beta.21072.1 16.9.0-beta.20628.1 16.0.461 diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs new file mode 100644 index 0000000000..1c65bb13d6 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollection +{ + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Runtime.Serialization; + + /// + /// Payload object that is used to exchange data between datacollector process and runner process. + /// + [DataContract] + public class AfterTestRunEndResult + { + /// + /// Initializes a new instance of the class. + /// + /// + /// The collection of attachment sets. + /// + /// + /// The metrics. + /// + public AfterTestRunEndResult(Collection attachmentSets, IDictionary metrics) + { + this.AttachmentSets = attachmentSets; + this.Metrics = metrics; + } + + [DataMember] + public Collection AttachmentSets { get; private set; } + + [DataMember] + public IDictionary Metrics { get; private set; } + } +} \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs index fe3d2aa44c..401782efd3 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs @@ -8,12 +8,12 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector using System.Collections.ObjectModel; using System.Globalization; using System.Linq; - using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; @@ -61,13 +61,18 @@ internal class DataCollectionManager : IDataCollectionManager /// private DataCollectorExtensionManager dataCollectorExtensionManager; + /// + /// Request data + /// + private IDataCollectionTelemetryManager dataCollectionTelemetryManager; + /// /// Initializes a new instance of the class. /// /// /// The message Sink. /// - internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollectionAttachmentManager(), messageSink) + internal DataCollectionManager(IMessageSink messageSink, IRequestData requestData) : this(new DataCollectionAttachmentManager(), messageSink, new DataCollectionTelemetryManager(requestData)) { } @@ -83,13 +88,14 @@ internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollecti /// /// The constructor is not public because the factory method should be used to get instances of this class. /// - protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink) + protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager) { this.attachmentManager = datacollectionAttachmentManager; this.messageSink = messageSink; this.events = new TestPlatformDataCollectionEvents(); this.dataCollectorExtensionManager = null; this.RunDataCollectors = new Dictionary(); + this.dataCollectionTelemetryManager = dataCollectionTelemetryManager; } /// @@ -128,7 +134,7 @@ private DataCollectorExtensionManager DataCollectorExtensionManager /// /// The . /// - public static DataCollectionManager Create(IMessageSink messageSink) + public static DataCollectionManager Create(IMessageSink messageSink, IRequestData requestData) { if (Instance == null) { @@ -136,7 +142,7 @@ public static DataCollectionManager Create(IMessageSink messageSink) { if (Instance == null) { - Instance = new DataCollectionManager(messageSink); + Instance = new DataCollectionManager(messageSink, requestData); } } } @@ -673,6 +679,8 @@ private void AddCollectorEnvironmentVariables( dataCollectionWrapper.Logger.LogError(this.dataCollectionEnvironmentContext.SessionDataCollectionContext, message); } } + + dataCollectionTelemetryManager.RecordEnvironmentVariableConflict(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value, alreadyRequestedVariable.Value); } else { @@ -685,6 +693,8 @@ private void AddCollectorEnvironmentVariables( dataCollectorEnvironmentVariables.Add( namevaluepair.Key, new DataCollectionEnvironmentVariable(namevaluepair, collectorFriendlyName)); + + dataCollectionTelemetryManager.RecordEnvironmentVariableAddition(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value); } } } diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs new file mode 100644 index 0000000000..9396a17dc0 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; +using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using System; +using System.Linq; + +namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector +{ + /// + /// Stores and provides telemetry information for data collection. + /// + internal class DataCollectionTelemetryManager : IDataCollectionTelemetryManager + { + private const string CorProfilerVariable = "COR_PROFILER"; + private const string CoreClrProfilerVariable = "CORECLR_PROFILER"; + private const string ClrIeInstrumentationMethodConfigurationPrefix32Variable = "MicrosoftInstrumentationEngine_ConfigPath32_"; + private const string ClrIeInstrumentationMethodConfigurationPrefix64Variable = "MicrosoftInstrumentationEngine_ConfigPath64_"; + + private static readonly Guid ClrIeProfilerGuid = Guid.Parse("{324f817a-7420-4e6d-b3c1-143fbed6d855}"); + private const string OverwrittenProfilerName = "overwritten"; + + private readonly IRequestData requestData; + + internal DataCollectionTelemetryManager(IRequestData requestData) + { + this.requestData = requestData; + } + + /// + public void RecordEnvironmentVariableAddition(DataCollectorInformation dataCollectorInformation, string name, string value) + { + RecordProfilerMetricForNewVariable(CorProfilerVariable, TelemetryDataConstants.DataCollectorsCorProfiler, dataCollectorInformation, name, value); + RecordProfilerMetricForNewVariable(CoreClrProfilerVariable, TelemetryDataConstants.DataCollectorsCoreClrProfiler, dataCollectorInformation, name, value); + } + + /// + public void RecordEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue) + { + RecordProfilerMetricForConflictedVariable(CorProfilerVariable, TelemetryDataConstants.DataCollectorsCorProfiler, dataCollectorInformation, name, value, existingValue); + RecordProfilerMetricForConflictedVariable(CoreClrProfilerVariable, TelemetryDataConstants.DataCollectorsCoreClrProfiler, dataCollectorInformation, name, value, existingValue); + } + + private void RecordProfilerMetricForNewVariable(string profilerVariable, string telemetryPrefix, DataCollectorInformation dataCollectorInformation, string name, string value) + { + if (!string.Equals(profilerVariable, name, StringComparison.Ordinal)) + { + return; + } + + requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), GetProfilerGuid(value).ToString()); + } + + private void RecordProfilerMetricForConflictedVariable(string profilerVariable, string telemetryPrefix, DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue) + { + // If data collector is requesting same profiler record it same as new + if (string.Equals(value, existingValue, StringComparison.OrdinalIgnoreCase)) + { + RecordProfilerMetricForNewVariable(profilerVariable, telemetryPrefix, dataCollectorInformation, name, value); + return; + } + + if (!string.Equals(profilerVariable, name, StringComparison.Ordinal)) + { + return; + } + + var existingProfilerGuid = GetProfilerGuid(existingValue); + + if (ClrIeProfilerGuid == existingProfilerGuid) + { + if (dataCollectorInformation.TestExecutionEnvironmentVariables != null && + dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix32Variable)) && + dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix64Variable))) + { + requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), ClrIeProfilerGuid.ToString()); + return; + } + } + + requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), $"{existingProfilerGuid}({OverwrittenProfilerName}:{GetProfilerGuid(value)})"); + } + + private static Guid GetProfilerGuid(string profilerGuid) + { + Guid guid; + if (Guid.TryParse(profilerGuid, out guid)) + { + return guid; + } + + return Guid.Empty; + } + + private static string GetTelemetryKey(string telemetryPrefix, DataCollectorInformation dataCollectorInformation) + { + return string.Format("{0}.{1}", telemetryPrefix, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()); + } + } +} diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs new file mode 100644 index 0000000000..9bc3550914 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces +{ + /// + /// The IDataCollectionTelemetryManager Interface. + /// + internal interface IDataCollectionTelemetryManager + { + /// + /// Record telemetry regarding environment variable added. + /// + /// + /// Data collector information which requested environment variable. + /// + /// + /// Environment variable name. + /// + /// + /// Environment variable value. + /// + void RecordEnvironmentVariableAddition(DataCollectorInformation dataCollectorInformation, string name, string value); + + /// + /// Record telemetry regarding environment variable is conflicting. + /// + /// + /// Data collector information which requested environment variable. + /// + /// + /// Environment variable name. + /// + /// + /// Environment variable value. + /// + /// + /// Environment variable value that was requested previously. + /// + void RecordEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue); + } +} diff --git a/src/Microsoft.TestPlatform.Common/RequestData.cs b/src/Microsoft.TestPlatform.Common/RequestData.cs index 14f2ae0033..128bc3fc5f 100644 --- a/src/Microsoft.TestPlatform.Common/RequestData.cs +++ b/src/Microsoft.TestPlatform.Common/RequestData.cs @@ -4,7 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common { using System; - + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; /// @@ -23,6 +23,15 @@ public class RequestData : IRequestData /// private ProtocolConfig protocolConfig; + /// + /// The default constructor for request data. + /// + public RequestData() + { + this.MetricsCollection = new NoOpMetricsCollection(); + this.IsTelemetryOptedIn = false; + } + /// /// Gets or sets the metrics collection. /// diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index 2063d1251f..0b1ca41424 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -26,6 +26,10 @@ public static class TelemetryDataConstants public static string DataCollectorsEnabled = "VS.TestRun.DataCollectorsEnabled"; + internal const string DataCollectorsCorProfiler = "VS.TestPlatform.DataCollector.CorProfiler"; + + internal const string DataCollectorsCoreClrProfiler = "VS.TestPlatform.DataCollector.CoreClrProfiler"; + public static string RunState = "VS.TestRun.RunState"; public static string NumberOfSourcesSentForRun = "VS.TestRun.NumberOfSources"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index ee319c2744..7add8c6308 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -16,11 +16,13 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollect using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; @@ -49,6 +51,8 @@ internal class DataCollectionRequestHandler : IDataCollectionRequestHandler, IDi private IFileHelper fileHelper; + private IRequestData requestData; + /// /// Use to cancel data collection test case events monitoring if test run is canceled. /// @@ -60,14 +64,18 @@ internal class DataCollectionRequestHandler : IDataCollectionRequestHandler, IDi /// /// The message sink. /// - protected DataCollectionRequestHandler(IMessageSink messageSink) + /// + /// The request data. + /// + protected DataCollectionRequestHandler(IMessageSink messageSink, IRequestData requestData) : this( new SocketCommunicationManager(), messageSink, - DataCollectionManager.Create(messageSink), + DataCollectionManager.Create(messageSink, requestData), new DataCollectionTestCaseEventHandler(), JsonDataSerializer.Instance, - new FileHelper()) + new FileHelper(), + requestData) { this.messageSink = messageSink; } @@ -93,13 +101,17 @@ protected DataCollectionRequestHandler(IMessageSink messageSink) /// /// File Helper /// + /// + /// Request data + /// protected DataCollectionRequestHandler( ICommunicationManager communicationManager, IMessageSink messageSink, IDataCollectionManager dataCollectionManager, IDataCollectionTestCaseEventHandler dataCollectionTestCaseEventHandler, IDataSerializer dataSerializer, - IFileHelper fileHelper) + IFileHelper fileHelper, + IRequestData requestData) { this.communicationManager = communicationManager; this.messageSink = messageSink; @@ -108,6 +120,7 @@ protected DataCollectionRequestHandler( this.dataCollectionTestCaseEventHandler = dataCollectionTestCaseEventHandler; this.cancellationTokenSource = new CancellationTokenSource(); this.fileHelper = fileHelper; + this.requestData = requestData; } /// @@ -139,13 +152,16 @@ public static DataCollectionRequestHandler Create( { if (Instance == null) { + var requestData = new RequestData(); + Instance = new DataCollectionRequestHandler( communicationManager, messageSink, - DataCollectionManager.Create(messageSink), + DataCollectionManager.Create(messageSink, requestData), new DataCollectionTestCaseEventHandler(), JsonDataSerializer.Instance, - new FileHelper()); + new FileHelper(), + requestData); } } } @@ -295,6 +311,7 @@ private void HandleBeforeTestRunStart(Message message) { // Initialize datacollectors and get environment variables. var payload = this.dataSerializer.DeserializePayload(message); + this.UpdateRequestData(payload.IsTelemetryOptedIn); this.AddExtensionAssemblies(payload); var envVariables = this.dataCollectionManager.InitializeDataCollectors(payload.SettingsXml); @@ -372,17 +389,27 @@ private void HandleAfterTestRunEnd(Message message) } var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); + var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, this.requestData.MetricsCollection.Metrics); // Dispose all datacollectors before sending attachments to vstest.console process. // As datacollector process exits itself on parent process(vstest.console) exits. this.dataCollectionManager?.Dispose(); - this.communicationManager.SendMessage(MessageType.AfterTestRunEndResult, attachmentsets); + this.communicationManager.SendMessage(MessageType.AfterTestRunEndResult, afterTestRunEndResult); EqtTrace.Info("DataCollectionRequestHandler.ProcessRequests : Session End message received from server. Closing the connection."); this.Close(); EqtTrace.Info("DataCollectionRequestHandler.ProcessRequests : DataCollection completed"); } + + private void UpdateRequestData(bool isTelemetryOptedIn) + { + if (isTelemetryOptedIn != this.requestData.IsTelemetryOptedIn) + { + this.requestData.MetricsCollection = isTelemetryOptedIn ? (IMetricsCollection)new MetricsCollection() : new NoOpMetricsCollection(); + this.requestData.IsTelemetryOptedIn = isTelemetryOptedIn; + } + } } } \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs index 0da7ea32d2..f51e3b7d5d 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs @@ -4,7 +4,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection { using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Globalization; using System.Net; @@ -108,7 +107,7 @@ public void SendTestHostLaunched(TestHostLaunchedPayload testHostLaunchedPayload } /// - public BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settingsXml, IEnumerable sources, ITestMessageEventHandler runEventsHandler) + public BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settingsXml, IEnumerable sources, bool isTelemetryOptedIn, ITestMessageEventHandler runEventsHandler) { var isDataCollectionStarted = false; BeforeTestRunStartResult result = null; @@ -121,7 +120,8 @@ public BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settin var payload = new BeforeTestRunStartPayload { SettingsXml = settingsXml, - Sources = sources + Sources = sources, + IsTelemetryOptedIn = isTelemetryOptedIn }; this.communicationManager.SendMessage(MessageType.BeforeTestRunStart, payload); @@ -151,10 +151,10 @@ public BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settin } /// - public Collection SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled) + public AfterTestRunEndResult SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled) { var isDataCollectionComplete = false; - Collection attachmentSets = null; + AfterTestRunEndResult result = null; if (EqtTrace.IsVerboseEnabled) { @@ -181,12 +181,12 @@ public Collection SendAfterTestRunEndAndGetResult(ITestMessageEve } else if (message.MessageType == MessageType.AfterTestRunEndResult) { - attachmentSets = this.dataSerializer.DeserializePayload>(message); + result = this.dataSerializer.DeserializePayload(message); isDataCollectionComplete = true; } } - return attachmentSets; + return result; } private void LogDataCollectorMessage(DataCollectionMessageEventArgs dataCollectionMessageEventArgs, ITestMessageEventHandler requestHandler) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataCollectionRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataCollectionRequestSender.cs index 4e70e5ea41..5dd36660b1 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataCollectionRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataCollectionRequestSender.cs @@ -4,11 +4,9 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection.Interfaces { using System.Collections.Generic; - using System.Collections.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Common.DataCollection; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; - using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; /// @@ -51,13 +49,16 @@ internal interface IDataCollectionRequestSender /// /// Test run sources /// + /// + /// Telemetry opted in flag. + /// /// /// Test message event handler for handling messages. /// /// /// BeforeTestRunStartResult containing environment variables /// - BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settingXml, IEnumerable sources, ITestMessageEventHandler runEventsHandler); + BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settingXml, IEnumerable sources, bool isTelemetryOptedIn, ITestMessageEventHandler runEventsHandler); /// /// Sends the AfterTestRunEnd event and waits for result @@ -69,8 +70,8 @@ internal interface IDataCollectionRequestSender /// The value to specify whether the test run is canceled or not. /// /// - /// DataCollector attachments + /// AfterTestRunEndResult containing dataCollector attachments and metrics /// - Collection SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled); + AfterTestRunEndResult SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled); } } \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/BeforeTestRunStartPayload.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/BeforeTestRunStartPayload.cs index c6150547fd..a8872140fe 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/BeforeTestRunStartPayload.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/BeforeTestRunStartPayload.cs @@ -19,5 +19,10 @@ public class BeforeTestRunStartPayload /// Gets or sets list of test sources. /// public IEnumerable Sources { get; set; } + + /// + /// Gets or sets a value indicating whether telemetry is enabled. + /// + public bool IsTelemetryOptedIn { get; set; } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs index 282ceb5bf5..999e57309e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs @@ -29,6 +29,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection using CrossPlatEngineResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources; using CommunicationUtilitiesResources = Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources; using CoreUtilitiesConstants = Microsoft.VisualStudio.TestPlatform.CoreUtilities.Constants; + using Microsoft.VisualStudio.TestPlatform.Common.DataCollection; /// /// Managed datacollector interaction from runner process. @@ -147,15 +148,24 @@ internal ProxyDataCollectionManager(IRequestData requestData, string settingsXml /// public Collection AfterTestRunEnd(bool isCanceled, ITestMessageEventHandler runEventsHandler) { - Collection attachmentSet = null; + AfterTestRunEndResult afterTestRunEnd = null; this.InvokeDataCollectionServiceAction( - () => - { - EqtTrace.Info("ProxyDataCollectionManager.AfterTestRunEnd: Get attachment set for datacollector processId: {0} port: {1}", dataCollectionProcessId, dataCollectionPort); - attachmentSet = this.dataCollectionRequestSender.SendAfterTestRunEndAndGetResult(runEventsHandler, isCanceled); - }, - runEventsHandler); - return attachmentSet; + () => + { + EqtTrace.Info("ProxyDataCollectionManager.AfterTestRunEnd: Get attachment set for datacollector processId: {0} port: {1}", dataCollectionProcessId, dataCollectionPort); + afterTestRunEnd = this.dataCollectionRequestSender.SendAfterTestRunEndAndGetResult(runEventsHandler, isCanceled); + }, + runEventsHandler); + + if (requestData.IsTelemetryOptedIn && afterTestRunEnd?.Metrics != null) + { + foreach (var metric in afterTestRunEnd.Metrics) + { + requestData.MetricsCollection.Add(metric.Key, metric.Value); + } + } + + return afterTestRunEnd?.AttachmentSets; } /// @@ -186,7 +196,7 @@ public DataCollectionParameters BeforeTestRunStart( () => { EqtTrace.Info("ProxyDataCollectionManager.BeforeTestRunStart: Get environment variable and port for datacollector processId: {0} port: {1}", this.dataCollectionProcessId, this.dataCollectionPort); - var result = this.dataCollectionRequestSender.SendBeforeTestRunStartAndGetResult(this.SettingsXml, this.Sources, runEventsHandler); + var result = this.dataCollectionRequestSender.SendBeforeTestRunStartAndGetResult(this.SettingsXml, this.Sources, this.requestData.IsTelemetryOptedIn, runEventsHandler); environmentVariables = result.EnvironmentVariables; dataCollectionEventsPort = result.DataCollectionEventsPort; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index a05b509d20..eb87327c37 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -19,14 +19,19 @@ public class CodeCoverageAcceptanceTestBase : AcceptanceTestBase */ protected const double ExpectedMinimalModuleCoverage = 30.0; - protected string GetCodeCoveragePath() + protected string GetNetStandardAdapterPath() { return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); } + protected string GetNetFrameworkAdapterPath() + { + return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "net451", "win7-x64", "Extensions"); + } + protected string GetCodeCoverageExePath() { - return Path.Combine(this.GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe"); + return Path.Combine(this.GetNetStandardAdapterPath(), "CodeCoverage", "CodeCoverage.exe"); } protected XmlNode GetModuleNode(XmlNode node, string name) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 94fabb2a4b..462201a076 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -18,7 +18,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests using Microsoft.VisualStudio.TestTools.UnitTesting; using Castle.Core.Internal; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + [TestClass] + //Code coverage only supported on windows (based on the message in output) + [TestCategory("Windows-Review")] public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper vstestConsoleWrapper; @@ -48,7 +52,7 @@ public void TestRunWithCodeCoverage(RunnerInfo runnerInfo) this.Setup(); // act - this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetCodeCoverageRunSettings(1), this.runEventHandler); + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetCodeCoverageRunSettings(1), new TestPlatformOptions { CollectMetrics = true }, this.runEventHandler); // assert Assert.AreEqual(6, this.runEventHandler.TestResults.Count); @@ -57,6 +61,33 @@ public void TestRunWithCodeCoverage(RunnerInfo runnerInfo) Assert.AreEqual(expectedNumberOfAttachments, this.runEventHandler.Attachments.Count); AssertCoverageResults(this.runEventHandler.Attachments); + + Assert.AreEqual("e5f256dc-7959-4dd6-8e4f-c11150ab28e0", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CorProfiler.datacollector://microsoft/CodeCoverage/2.0"]); + Assert.AreEqual("e5f256dc-7959-4dd6-8e4f-c11150ab28e0", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CoreClrProfiler.datacollector://microsoft/CodeCoverage/2.0"]); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void TestRunWithCodeCoverageUsingClrIe(RunnerInfo runnerInfo) + { + // arrange + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + // act + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetCodeCoverageRunSettings(1, true), new TestPlatformOptions { CollectMetrics = true }, this.runEventHandler); + + // assert + Assert.AreEqual(6, this.runEventHandler.TestResults.Count); + + int expectedNumberOfAttachments = 1; + Assert.AreEqual(expectedNumberOfAttachments, this.runEventHandler.Attachments.Count); + + AssertCoverageResults(this.runEventHandler.Attachments); + + Assert.AreEqual("324f817a-7420-4e6d-b3c1-143fbed6d855", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CorProfiler.datacollector://microsoft/CodeCoverage/2.0"]); + Assert.AreEqual("324f817a-7420-4e6d-b3c1-143fbed6d855", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CoreClrProfiler.datacollector://microsoft/CodeCoverage/2.0"]); } [TestMethod] @@ -69,13 +100,16 @@ public void TestRunWithCodeCoverageParallel(RunnerInfo runnerInfo) this.Setup(); // act - this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetCodeCoverageRunSettings(4), this.runEventHandler); + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetCodeCoverageRunSettings(4), new TestPlatformOptions { CollectMetrics = true }, this.runEventHandler); // assert Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(1, this.runEventHandler.Attachments.Count); AssertCoverageResults(this.runEventHandler.Attachments); + + Assert.AreEqual("e5f256dc-7959-4dd6-8e4f-c11150ab28e0", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CorProfiler.datacollector://microsoft/CodeCoverage/2.0"]); + Assert.AreEqual("e5f256dc-7959-4dd6-8e4f-c11150ab28e0", this.runEventHandler.Metrics["VS.TestPlatform.DataCollector.CoreClrProfiler.datacollector://microsoft/CodeCoverage/2.0"]); } [TestMethod] @@ -124,7 +158,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForAttachmentsProcessing)); Assert.IsTrue(File.Exists(runEventHandler.Attachments.First().Attachments.First().Uri.LocalPath)); - Assert.IsTrue(File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath) != testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)); + Assert.IsTrue(!File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath)); } [TestMethod] @@ -170,7 +204,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(Runne Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.IsNullOrEmpty()); Assert.IsTrue(File.Exists(runEventHandler.Attachments.First().Attachments.First().Uri.LocalPath)); - Assert.IsTrue(File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath) != testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)); + Assert.IsTrue(!File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath)); } [TestMethod] @@ -220,76 +254,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingModuleDuplicate Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForAttachmentsProcessing)); Assert.IsTrue(File.Exists(runEventHandler.Attachments.First().Attachments.First().Uri.LocalPath)); - Assert.IsTrue(File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath) != testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)); - } - - [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public async Task TestRunWithCodeCoverageAndAttachmentsProcessingCancelled(RunnerInfo runnerInfo) - { - // arrange - AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); - this.Setup(); - - if (!testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)) return; - - this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler); - Assert.AreEqual(3, this.runEventHandler.TestResults.Count); - Assert.AreEqual(1, this.runEventHandler.Attachments.Count); - - List attachments = Enumerable.Range(0, 1000).Select(i => this.runEventHandler.Attachments.First()).ToList(); - - CancellationTokenSource cts = new CancellationTokenSource(); - - Task attachmentsProcessing = this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(attachments, null, true, true, testRunAttachmentsProcessingEventHandler, cts.Token); - - while (true) - { - try - { - if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count >= 3) - break; - } - catch - { - // ignore - } - await Task.Delay(100); - } - - // act - cts.Cancel(); - - // Assert - await attachmentsProcessing; - testRunAttachmentsProcessingEventHandler.EnsureSuccess(); - - Assert.AreEqual(1000, this.testRunAttachmentsProcessingEventHandler.Attachments.Count); - - Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.IsCanceled); - Assert.IsNull(testRunAttachmentsProcessingEventHandler.CompleteArgs.Error); - - Assert.IsTrue(3 <= testRunAttachmentsProcessingEventHandler.ProgressArgs.Count); - for (int i = 0; i < testRunAttachmentsProcessingEventHandler.ProgressArgs.Count; i++) - { - VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingProgressEventArgs progressArgs = testRunAttachmentsProcessingEventHandler.ProgressArgs[i]; - Assert.AreEqual(1, progressArgs.CurrentAttachmentProcessorIndex); - Assert.AreEqual("datacollector://microsoft/CodeCoverage/2.0", progressArgs.CurrentAttachmentProcessorUris.First().AbsoluteUri); - Assert.AreEqual(1, progressArgs.AttachmentProcessorsCount); - - if (i == 0) - { - Assert.AreEqual(0, progressArgs.CurrentAttachmentProcessorProgress); - } - } - - Assert.AreEqual("Canceled", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]); - Assert.AreEqual(1000L, testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsSentForProcessing]); - Assert.AreEqual(1000L, testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsAfterProcessing]); - Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForAttachmentsProcessing)); - - Assert.IsTrue(File.Exists(runEventHandler.Attachments.First().Attachments.First().Uri.LocalPath)); + Assert.IsTrue(!File.Exists(runEventHandler.Attachments.Last().Attachments.First().Uri.LocalPath)); } [TestMethod] @@ -333,19 +298,21 @@ private IList GetProjects() /// Default RunSettings /// /// - private string GetCodeCoverageRunSettings(int cpuCount) + private string GetCodeCoverageRunSettings(int cpuCount, bool useClrIeInstrumentationEngine = false) { string runSettingsXml = $@" {FrameworkArgValue} - {this.GetCodeCoveragePath()} + {this.GetNetStandardAdapterPath()} {cpuCount} + {useClrIeInstrumentationEngine} + {useClrIeInstrumentationEngine} diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestHandlerTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestHandlerTests.cs index 08779a496b..98c6473770 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestHandlerTests.cs @@ -14,12 +14,14 @@ namespace Microsoft.TestPlatform.CommunicationUtilities.UnitTests using Microsoft.VisualStudio.TestPlatform.Common.DataCollection; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization; using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -42,6 +44,8 @@ public class DataCollectionRequestHandlerTests private TestableDataCollectionRequestHandler requestHandler; private Mock mockDataSerializer; private Mock mockFileHelper; + private Mock mockRequestData; + private Mock mockMetricsCollection; private Message afterTestRunEnd = new Message() { MessageType = MessageType.AfterTestRunEnd, Payload = "false" }; private Message beforeTestRunStart = new Message() { @@ -58,7 +62,10 @@ public DataCollectionRequestHandlerTests() this.mockDataCollectionTestCaseEventHandler = new Mock(); this.mockDataCollectionTestCaseEventHandler.Setup(x => x.WaitForRequestHandlerConnection(It.IsAny())).Returns(true); this.mockFileHelper = new Mock(); - this.requestHandler = new TestableDataCollectionRequestHandler(this.mockCommunicationManager.Object, this.mockMessageSink.Object, this.mockDataCollectionManager.Object, this.mockDataCollectionTestCaseEventHandler.Object, this.mockDataSerializer.Object, this.mockFileHelper.Object); + this.mockRequestData = new Mock(); + this.mockMetricsCollection = new Mock(); + this.mockRequestData.Setup(r => r.MetricsCollection).Returns(this.mockMetricsCollection.Object); + this.requestHandler = new TestableDataCollectionRequestHandler(this.mockCommunicationManager.Object, this.mockMessageSink.Object, this.mockDataCollectionManager.Object, this.mockDataCollectionTestCaseEventHandler.Object, this.mockDataSerializer.Object, this.mockFileHelper.Object, this.mockRequestData.Object); this.mockCommunicationManager.SetupSequence(x => x.ReceiveMessage()).Returns(this.beforeTestRunStart).Returns(this.afterTestRunEnd); @@ -217,7 +224,7 @@ public void ProcessRequestsShouldProcessRequests() // Verify AfterTestRun events. this.mockDataCollectionManager.Verify(x => x.SessionEnded(It.IsAny()), Times.Once); - this.mockCommunicationManager.Verify(x => x.SendMessage(MessageType.AfterTestRunEndResult, It.IsAny>()), Times.Once); + this.mockCommunicationManager.Verify(x => x.SendMessage(MessageType.AfterTestRunEndResult, It.IsAny()), Times.Once); } [TestMethod] @@ -371,5 +378,50 @@ public void ProcessRequestShouldCallSessionStartWithCorrectTestSources() y => y.GetPropertyValue>("TestSources").Contains("test1.dll") && y.GetPropertyValue>("TestSources").Contains("test2.dll")))); } + + [TestMethod] + public void ProcessRequestShouldEnableTelemetry() + { + var beforeTestRunStartPayload = new BeforeTestRunStartPayload { SettingsXml = "settingsxml", Sources = new List { "test1.dll", "test2.dll" }, IsTelemetryOptedIn = true }; + this.mockRequestData.Setup(r => r.IsTelemetryOptedIn).Returns(false); + this.mockDataSerializer.Setup(x => x.DeserializePayload(It.Is(y => y.MessageType == MessageType.BeforeTestRunStart))) + .Returns(beforeTestRunStartPayload); + var message = new Message() { MessageType = MessageType.BeforeTestRunStart, Payload = JToken.FromObject(beforeTestRunStartPayload) }; + this.mockCommunicationManager.SetupSequence(x => x.ReceiveMessage()).Returns(message).Returns(this.afterTestRunEnd); + this.requestHandler.ProcessRequests(); + + this.mockRequestData.VerifySet(r => r.IsTelemetryOptedIn = true); + this.mockRequestData.VerifySet(r => r.MetricsCollection = It.IsAny()); + } + + [TestMethod] + public void ProcessRequestShouldNotEnableTelemetryIfTelemetryEnabled() + { + var beforeTestRunStartPayload = new BeforeTestRunStartPayload { SettingsXml = "settingsxml", Sources = new List { "test1.dll", "test2.dll" }, IsTelemetryOptedIn = true }; + this.mockRequestData.Setup(r => r.IsTelemetryOptedIn).Returns(true); + this.mockDataSerializer.Setup(x => x.DeserializePayload(It.Is(y => y.MessageType == MessageType.BeforeTestRunStart))) + .Returns(beforeTestRunStartPayload); + var message = new Message() { MessageType = MessageType.BeforeTestRunStart, Payload = JToken.FromObject(beforeTestRunStartPayload) }; + this.mockCommunicationManager.SetupSequence(x => x.ReceiveMessage()).Returns(message).Returns(this.afterTestRunEnd); + this.requestHandler.ProcessRequests(); + + this.mockRequestData.VerifySet(r => r.IsTelemetryOptedIn = It.IsAny(), Times.Never); + this.mockRequestData.VerifySet(r => r.MetricsCollection = It.IsAny(), Times.Never); + } + + [TestMethod] + public void ProcessRequestShouldNotEnableTelemetryIfTelemetryEnablingNotRequested() + { + var beforeTestRunStartPayload = new BeforeTestRunStartPayload { SettingsXml = "settingsxml", Sources = new List { "test1.dll", "test2.dll" }, IsTelemetryOptedIn = false }; + this.mockRequestData.Setup(r => r.IsTelemetryOptedIn).Returns(false); + this.mockDataSerializer.Setup(x => x.DeserializePayload(It.Is(y => y.MessageType == MessageType.BeforeTestRunStart))) + .Returns(beforeTestRunStartPayload); + var message = new Message() { MessageType = MessageType.BeforeTestRunStart, Payload = JToken.FromObject(beforeTestRunStartPayload) }; + this.mockCommunicationManager.SetupSequence(x => x.ReceiveMessage()).Returns(message).Returns(this.afterTestRunEnd); + this.requestHandler.ProcessRequests(); + + this.mockRequestData.VerifySet(r => r.IsTelemetryOptedIn = It.IsAny(), Times.Never); + this.mockRequestData.VerifySet(r => r.MetricsCollection = It.IsAny(), Times.Never); + } } } \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs index 74381f5136..1e1c9cca49 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs @@ -6,7 +6,7 @@ namespace Microsoft.TestPlatform.CommunicationUtilities.UnitTests using System; using System.Collections.Generic; using System.Collections.ObjectModel; - + using Microsoft.VisualStudio.TestPlatform.Common.DataCollection; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -39,17 +39,21 @@ public void SendAfterTestRunEndAndGetResultShouldReturnAttachments() var attachment = new AttachmentSet(datacollectorUri, displayName); attachment.Attachments.Add(new UriDataAttachment(attachmentUri, "filename.txt")); - this.mockDataSerializer.Setup(x => x.DeserializePayload>(It.IsAny())).Returns(new Collection() { attachment }); + this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())).Returns( + new AfterTestRunEndResult(new Collection() { attachment }, new Dictionary())); this.mockCommunicationManager.Setup(x => x.ReceiveMessage()).Returns(new Message() { MessageType = MessageType.AfterTestRunEndResult, Payload = null }); - var attachmentSets = this.requestSender.SendAfterTestRunEndAndGetResult(null, false); + var result = this.requestSender.SendAfterTestRunEndAndGetResult(null, false); - Assert.IsNotNull(attachmentSets); - Assert.AreEqual(1, attachmentSets.Count); - Assert.IsNotNull(attachmentSets[0]); - Assert.AreEqual(displayName, attachmentSets[0].DisplayName); - Assert.AreEqual(datacollectorUri, attachmentSets[0].Uri); - Assert.AreEqual(attachmentUri, attachmentSets[0].Attachments[0].Uri); + Assert.IsNotNull(result); + Assert.IsNotNull(result.AttachmentSets); + Assert.IsNotNull(result.Metrics); + Assert.AreEqual(1, result.AttachmentSets.Count); + Assert.AreEqual(0, result.Metrics.Count); + Assert.IsNotNull(result.AttachmentSets[0]); + Assert.AreEqual(displayName, result.AttachmentSets[0].DisplayName); + Assert.AreEqual(datacollectorUri, result.AttachmentSets[0].Uri); + Assert.AreEqual(attachmentUri, result.AttachmentSets[0].Attachments[0].Uri); } [TestMethod] @@ -65,9 +69,9 @@ public void SendBeforeTestRunStartAndGetResultShouldSendBeforeTestRunStartMessag { var testSources = new List() { "test1.dll" }; this.mockCommunicationManager.Setup(x => x.ReceiveMessage()).Returns(new Message() { MessageType = MessageType.BeforeTestRunStartResult, Payload = null }); - this.requestSender.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, null); + this.requestSender.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, true, null); - this.mockCommunicationManager.Verify(x => x.SendMessage(MessageType.BeforeTestRunStart, It.IsAny())); + this.mockCommunicationManager.Verify(x => x.SendMessage(MessageType.BeforeTestRunStart, It.Is(p => p.SettingsXml == string.Empty && p.IsTelemetryOptedIn))); } } } diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestDoubles/TestableDataCollectionRequestHandler.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestDoubles/TestableDataCollectionRequestHandler.cs index 375272f712..b613e78c21 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestDoubles/TestableDataCollectionRequestHandler.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestDoubles/TestableDataCollectionRequestHandler.cs @@ -6,6 +6,7 @@ namespace Microsoft.TestPlatform.CommunicationUtilities.UnitTests.TestDoubles using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; /// @@ -13,8 +14,8 @@ namespace Microsoft.TestPlatform.CommunicationUtilities.UnitTests.TestDoubles /// internal class TestableDataCollectionRequestHandler : DataCollectionRequestHandler { - public TestableDataCollectionRequestHandler(ICommunicationManager communicationManager, IMessageSink messageSink, IDataCollectionManager dataCollectionManager, IDataCollectionTestCaseEventHandler dataCollectionTestCaseEventHandler, IDataSerializer dataSerializer, IFileHelper fIleHelper) - : base(communicationManager, messageSink, dataCollectionManager, dataCollectionTestCaseEventHandler, dataSerializer, fIleHelper) + public TestableDataCollectionRequestHandler(ICommunicationManager communicationManager, IMessageSink messageSink, IDataCollectionManager dataCollectionManager, IDataCollectionTestCaseEventHandler dataCollectionTestCaseEventHandler, IDataSerializer dataSerializer, IFileHelper fIleHelper, IRequestData requestData) + : base(communicationManager, messageSink, dataCollectionManager, dataCollectionTestCaseEventHandler, dataSerializer, fIleHelper, requestData) { } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyExecutionManagerWithDataCollectionTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyExecutionManagerWithDataCollectionTests.cs index 74a7c9a2b3..b8e2819084 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyExecutionManagerWithDataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyExecutionManagerWithDataCollectionTests.cs @@ -99,7 +99,7 @@ public void InitializeShouldSaveExceptionMessagesIfThrownByDataCollectionProcess { var mockRequestSender = new Mock(); var testSources = new List() { "abc.dll", "efg.dll" }; - mockRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, It.IsAny())).Throws(new Exception("MyException")); + mockRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, It.IsAny(), It.IsAny())).Throws(new Exception("MyException")); mockRequestSender.Setup(x => x.WaitForRequestHandlerConnection(It.IsAny())).Returns(true); var mockDataCollectionLauncher = new Mock(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs index c89ffb9fa8..e69707fd9c 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs @@ -156,16 +156,17 @@ public void BeforeTestRunStartShouldPassRunSettingsWithExtensionsFolderUpdatedAs string runsettings = $""; var sourceList = new List() { "testsource1.dll" }; this.proxyDataCollectionManager = new ProxyDataCollectionManager(this.mockRequestData.Object, runsettings, sourceList, this.mockDataCollectionRequestSender.Object, this.mockProcessHelper.Object, this.mockDataCollectionLauncher.Object); + this.mockRequestData.Setup(r => r.IsTelemetryOptedIn).Returns(true); BeforeTestRunStartResult res = new BeforeTestRunStartResult(new Dictionary(), 123); - this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), It.IsAny>(), It.IsAny())).Returns(res); + this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns(res); var result = this.proxyDataCollectionManager.BeforeTestRunStart(true, true, null); var extensionsFolderPath = Path.Combine(Path.GetDirectoryName(typeof(ITestPlatform).GetTypeInfo().Assembly.Location), "Extensions"); var expectedSettingsXML = $"{extensionsFolderPath}"; this.mockDataCollectionRequestSender.Verify( - x => x.SendBeforeTestRunStartAndGetResult(expectedSettingsXML, sourceList, It.IsAny()), Times.Once); + x => x.SendBeforeTestRunStartAndGetResult(expectedSettingsXML, sourceList, true, It.IsAny()), Times.Once); } [TestMethod] @@ -173,12 +174,12 @@ public void BeforeTestRunStartShouldReturnDataCollectorParameters() { BeforeTestRunStartResult res = new BeforeTestRunStartResult(new Dictionary(), 123); var sourceList = new List() { "testsource1.dll" }; - this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), It.IsAny>(), It.IsAny())).Returns(res); + this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns(res); var result = this.proxyDataCollectionManager.BeforeTestRunStart(true, true, null); this.mockDataCollectionRequestSender.Verify( - x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), sourceList, It.IsAny()), Times.Once); + x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), sourceList, false, It.IsAny()), Times.Once); Assert.IsNotNull(result); Assert.AreEqual(res.DataCollectionEventsPort, result.DataCollectionEventsPort); Assert.AreEqual(res.EnvironmentVariables.Count, result.EnvironmentVariables.Count); @@ -189,7 +190,7 @@ public void BeforeTestRunStartsShouldInvokeRunEventsHandlerIfExceptionIsThrown() { var mockRunEventsHandler = new Mock(); this.mockDataCollectionRequestSender.Setup( - x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), new List() { "testsource1.dll" }, It.IsAny())) + x => x.SendBeforeTestRunStartAndGetResult(It.IsAny(), new List() { "testsource1.dll" }, false, It.IsAny())) .Throws(); var result = this.proxyDataCollectionManager.BeforeTestRunStart(true, true, mockRunEventsHandler.Object); @@ -207,27 +208,35 @@ public void SendBeforeTestRunStartAndGetResultShouldBeInvokedWithCorrectTestSour this.proxyDataCollectionManager = new ProxyDataCollectionManager(this.mockRequestData.Object, string.Empty, testSources, this.mockDataCollectionRequestSender.Object, this.mockProcessHelper.Object, this.mockDataCollectionLauncher.Object); BeforeTestRunStartResult res = new BeforeTestRunStartResult(new Dictionary(), 123); - this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, It.IsAny())).Returns(res); + this.mockDataCollectionRequestSender.Setup(x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, It.IsAny(), It.IsAny())).Returns(res); var result = this.proxyDataCollectionManager.BeforeTestRunStart(true, true, null); this.mockDataCollectionRequestSender.Verify( - x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, It.IsAny()), Times.Once); + x => x.SendBeforeTestRunStartAndGetResult(string.Empty, testSources, false, It.IsAny()), Times.Once); Assert.IsNotNull(result); Assert.AreEqual(res.DataCollectionEventsPort, result.DataCollectionEventsPort); Assert.AreEqual(res.EnvironmentVariables.Count, result.EnvironmentVariables.Count); } [TestMethod] - public void AfterTestRunEndShouldReturnAttachments() + [DataRow(false)] + [DataRow(true)] + public void AfterTestRunEndShouldReturnAttachments(bool telemetryOptedIn) { var attachments = new Collection(); var dispName = "MockAttachments"; var uri = new Uri("Mock://Attachments"); var attachmentSet = new AttachmentSet(uri, dispName); attachments.Add(attachmentSet); + this.mockRequestData.Setup(m => m.IsTelemetryOptedIn).Returns(telemetryOptedIn); - this.mockDataCollectionRequestSender.Setup(x => x.SendAfterTestRunEndAndGetResult(It.IsAny(), It.IsAny())).Returns(attachments); + var metrics = new Dictionary() + { + {"key", "value"} + }; + + this.mockDataCollectionRequestSender.Setup(x => x.SendAfterTestRunEndAndGetResult(It.IsAny(), It.IsAny())).Returns(new AfterTestRunEndResult(attachments, metrics)); var result = this.proxyDataCollectionManager.AfterTestRunEnd(false, null); @@ -236,6 +245,15 @@ public void AfterTestRunEndShouldReturnAttachments() Assert.IsNotNull(result[0]); Assert.AreEqual(dispName, result[0].DisplayName); Assert.AreEqual(uri, result[0].Uri); + + if (telemetryOptedIn) + { + mockMetricsCollection.Verify(m => m.Add("key", "value"), Times.Once); + } + else + { + mockMetricsCollection.Verify(m => m.Add(It.IsAny(), It.IsAny()), Times.Never); + } } [TestMethod] diff --git a/test/datacollector.UnitTests/DataCollectionManagerTests.cs b/test/datacollector.UnitTests/DataCollectionManagerTests.cs index 81e98d6b95..26828281eb 100644 --- a/test/datacollector.UnitTests/DataCollectionManagerTests.cs +++ b/test/datacollector.UnitTests/DataCollectionManagerTests.cs @@ -32,6 +32,7 @@ public class DataCollectionManagerTests private List> envVarList; private List> codeCoverageEnvVarList; private Mock mockDataCollectionAttachmentManager; + private Mock mockDataCollectionTelemetryManager; public DataCollectionManagerTests() { @@ -47,8 +48,9 @@ public DataCollectionManagerTests() this.mockMessageSink = new Mock(); this.mockDataCollectionAttachmentManager = new Mock(); this.mockDataCollectionAttachmentManager.SetReturnsDefault>(new List()); + this.mockDataCollectionTelemetryManager = new Mock(); - this.dataCollectionManager = new TestableDataCollectionManager(this.mockDataCollectionAttachmentManager.Object, this.mockMessageSink.Object, this.mockDataCollector.Object, this.mockCodeCoverageDataCollector.Object); + this.dataCollectionManager = new TestableDataCollectionManager(this.mockDataCollectionAttachmentManager.Object, this.mockMessageSink.Object, this.mockDataCollector.Object, this.mockCodeCoverageDataCollector.Object, this.mockDataCollectionTelemetryManager.Object); } [TestMethod] @@ -183,6 +185,8 @@ public void InitializeDataCollectorsShouldLoadDataCollectorAndReturnEnvironmentV var result = this.dataCollectionManager.InitializeDataCollectors(this.dataCollectorSettings); Assert.AreEqual("value", result["key"]); + + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableAddition(It.IsAny(), "key", "value")); } [TestMethod] @@ -223,14 +227,19 @@ public void InitializeDataCollectorsShouldReturnFirstEnvironmentVariableIfMoreTh var result = this.dataCollectionManager.InitializeDataCollectors(this.dataCollectorSettings); Assert.AreEqual("value", result["key"]); + + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableAddition(It.IsAny(), "key", "value")); + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableConflict(It.IsAny(), "key", "value1", "value")); } [TestMethod] public void InitializeDataCollectorsShouldReturnOtherThanCodeCoverageEnvironmentVariableIfMoreThanOneVariablesWithSameKeyIsSpecified() { this.envVarList.Add(new KeyValuePair("cor_profiler", "clrie")); + this.envVarList.Add(new KeyValuePair("same_key", "same_value")); this.codeCoverageEnvVarList.Add(new KeyValuePair("cor_profiler", "direct")); this.codeCoverageEnvVarList.Add(new KeyValuePair("clrie_profiler_vanguard", "path")); + this.codeCoverageEnvVarList.Add(new KeyValuePair("same_key", "same_value")); this.dataCollectorSettings = string.Format(this.defaultRunSettings, string.Format(this.defaultDataCollectionSettings, "Code Coverage", "my://custom/ccdatacollector", this.mockDataCollector.Object.GetType().AssemblyQualifiedName, typeof(DataCollectionManagerTests).GetTypeInfo().Assembly.Location, string.Empty) + @@ -238,9 +247,16 @@ public void InitializeDataCollectorsShouldReturnOtherThanCodeCoverageEnvironment var result = this.dataCollectionManager.InitializeDataCollectors(this.dataCollectorSettings); - Assert.AreEqual(2, result.Count); + Assert.AreEqual(3, result.Count); Assert.AreEqual("clrie", result["cor_profiler"]); Assert.AreEqual("path", result["clrie_profiler_vanguard"]); + Assert.AreEqual("same_value", result["same_key"]); + + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableAddition(It.Is(i => i.DataCollectorConfig.FriendlyName == this.friendlyName), "cor_profiler", "clrie")); + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableConflict(It.Is(i => i.DataCollectorConfig.FriendlyName == "Code Coverage"), "cor_profiler", "direct", "clrie")); + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableAddition(It.Is(i => i.DataCollectorConfig.FriendlyName == "Code Coverage"), "clrie_profiler_vanguard", "path")); + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableAddition(It.Is(i => i.DataCollectorConfig.FriendlyName == this.friendlyName), "same_key", "same_value")); + this.mockDataCollectionTelemetryManager.Verify(tm => tm.RecordEnvironmentVariableConflict(It.Is(i => i.DataCollectorConfig.FriendlyName == "Code Coverage"), "same_key", "same_value", "same_value")); } [TestMethod] @@ -483,13 +499,13 @@ internal class TestableDataCollectionManager : DataCollectionManager DataCollector dataCollector; DataCollector ccDataCollector; - public TestableDataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, DataCollector dataCollector, DataCollector ccDataCollector) : this(datacollectionAttachmentManager, messageSink) + public TestableDataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, DataCollector dataCollector, DataCollector ccDataCollector, IDataCollectionTelemetryManager dataCollectionTelemetryManager) : this(datacollectionAttachmentManager, messageSink, dataCollectionTelemetryManager) { this.dataCollector = dataCollector; this.ccDataCollector = ccDataCollector; } - internal TestableDataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink) : base(datacollectionAttachmentManager, messageSink) + internal TestableDataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager) : base(datacollectionAttachmentManager, messageSink, dataCollectionTelemetryManager) { } diff --git a/test/datacollector.UnitTests/DataCollectionTelemetryManagerTests.cs b/test/datacollector.UnitTests/DataCollectionTelemetryManagerTests.cs new file mode 100644 index 0000000000..a0fbdb3bd2 --- /dev/null +++ b/test/datacollector.UnitTests/DataCollectionTelemetryManagerTests.cs @@ -0,0 +1,228 @@ +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.UnitTests; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Generic; + +namespace Microsoft.VisualStudio.TestPlatform.DataCollector.UnitTests +{ + [TestClass] + public class DataCollectionTelemetryManagerTests + { + private readonly Mock mockRequestData; + private readonly Mock mockMetricsCollection; + private readonly DataCollectionTelemetryManager telemetryManager; + private readonly DataCollectorInformation dataCollectorInformation; + + public DataCollectionTelemetryManagerTests() + { + this.mockRequestData = new Mock(); + this.mockMetricsCollection = new Mock(); + this.mockRequestData.Setup(m => m.MetricsCollection).Returns(this.mockMetricsCollection.Object); + + var dataCollectorMock = new Mock(); + var evnVariablesMock = dataCollectorMock.As(); + evnVariablesMock.Setup(a => a.GetTestExecutionEnvironmentVariables()).Returns(new KeyValuePair[] + { + new KeyValuePair("MicrosoftInstrumentationEngine_ConfigPath32_VanguardInstrumentationProfiler", "path1"), + new KeyValuePair("MicrosoftInstrumentationEngine_ConfigPath64_VanguardInstrumentationProfiler", "path2") + }); + + this.dataCollectorInformation = new DataCollectorInformation( + dataCollectorMock.Object, + null, + new DataCollectorConfig(typeof(CustomDataCollector)), + null, + new Mock().Object, + new TestPlatformDataCollectionEvents(), + new Mock().Object, + string.Empty); + + this.telemetryManager = new DataCollectionTelemetryManager(this.mockRequestData.Object); + } + + [TestMethod] + public void RecordEnvironmentVariableAddition_ShouldDoNothing_IfNotProfilerVariable() + { + // act + this.telemetryManager.RecordEnvironmentVariableAddition(this.dataCollectorInformation, "key", "value"); + + // assert + this.mockMetricsCollection.Verify(c => c.Add(It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public void RecordEnvironmentVariableConflict_ShouldDoNothing_IfNotProfilerVariable_ValuesSame() + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "key", "value", "value"); + + // assert + this.mockMetricsCollection.Verify(c => c.Add(It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public void RecordEnvironmentVariableConflict_ShouldDoNothing_IfNotProfilerVariable_ValuesDifferent() + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "key", "value", "othervalue"); + + // assert + this.mockMetricsCollection.Verify(c => c.Add(It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableAddition_ShouldCollectTelemetry_IfCorProfilerVariable(string profilerGuid, string profilerName) + { + // act + this.telemetryManager.RecordEnvironmentVariableAddition(this.dataCollectorInformation, "COR_PROFILER", profilerGuid); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CorProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", profilerName), Times.Once); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableAddition_ShouldCollectTelemetry_IfCoreClrProfilerVariable(string profilerGuid, string profilerName) + { + // act + this.telemetryManager.RecordEnvironmentVariableAddition(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CoreClrProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", profilerName), Times.Once); + } + + [TestMethod] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableConflict_ShouldCollectOverwrittenTelemetry_IfCorProfilerVariable(string existingProfilerGuid, string profilerGuid, string expectedOverwrittenProfiler) + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "COR_PROFILER", profilerGuid, existingProfilerGuid); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CorProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", $"{Guid.Parse(existingProfilerGuid)}(overwritten:{expectedOverwrittenProfiler})"), Times.Once); + } + + [TestMethod] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{0f8fad5b-d9cb-469f-a165-70867728950e}", "{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableConflict_ShouldCollectOverwrittenTelemetry_IfCoreClrProfilerVariable(string existingProfilerGuid, string profilerGuid, string expectedOverwrittenProfiler) + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid, existingProfilerGuid); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CoreClrProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", $"{Guid.Parse(existingProfilerGuid)}(overwritten:{expectedOverwrittenProfiler})"), Times.Once); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}")] + [DataRow("AAAAAAAAAAAAA")] + public void RecordEnvironmentVariableConflict_ShouldCollectClrIeTelemetry_IfCorProfilerVariableAndCollectorSpecifiesClrIeProfile(string profilerGuid) + { + // arrange + this.dataCollectorInformation.SetTestExecutionEnvironmentVariables(); + + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "COR_PROFILER", profilerGuid, "{324F817A-7420-4E6D-B3C1-143FBED6D855}"); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CorProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", "324f817a-7420-4e6d-b3c1-143fbed6d855"), Times.Once); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}")] + [DataRow("AAAAAAAAAAAAA")] + public void RecordEnvironmentVariableConflict_ShouldCollectClrIeTelemetry_IfCoreClrProfilerVariableAndCollectorSpecifiesClrIeProfile(string profilerGuid) + { + // arrange + this.dataCollectorInformation.SetTestExecutionEnvironmentVariables(); + + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid, "{324F817A-7420-4E6D-B3C1-143FBED6D855}"); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CoreClrProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", "324f817a-7420-4e6d-b3c1-143fbed6d855"), Times.Once); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableConflict_ShouldCollectTelemetry_IfCorProfilerVariableAndBothValuesSame(string profilerGuid, string profilerName) + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "COR_PROFILER", profilerGuid, profilerGuid.ToLower()); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CorProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", profilerName), Times.Once); + } + + [TestMethod] + [DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "e5f256dc-7959-4dd6-8e4f-c11150ab28e0")] + [DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "324f817a-7420-4e6d-b3c1-143fbed6d855")] + [DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "9317ae81-bcd8-47b7-aaa1-a28062e41c71")] + [DataRow("AAAAAAAAAAAAA", "00000000-0000-0000-0000-000000000000")] + public void RecordEnvironmentVariableConflict_ShouldCollectTelemetry_IfCoreClrProfilerVariableAndBothValuesSame(string profilerGuid, string profilerName) + { + // act + this.telemetryManager.RecordEnvironmentVariableConflict(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid, profilerGuid.ToUpper()); + + // assert + this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.CoreClrProfiler.{dataCollectorInformation.DataCollectorConfig.TypeUri}", profilerName), Times.Once); + } + } +}