chore(playwrighttesting): add logging per framework and fix auth handling in vstest

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/api/Azure.Developer.MicrosoftPlaywrightTesting.NUnit.netstandard2.0.cs

@@ -3,7 +3,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit
     [NUnit.Framework.SetUpFixtureAttribute]
     public partial class PlaywrightServiceNUnit : Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightService
     {
-        public PlaywrightServiceNUnit(Azure.Core.TokenCredential? credential = null) : base (default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions), default(Azure.Core.TokenCredential)) { }
+        public PlaywrightServiceNUnit(Azure.Core.TokenCredential? credential = null) : base (default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions), default(Azure.Core.TokenCredential), default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger)) { }
         public static Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions { get { throw null; } }
         [NUnit.Framework.OneTimeSetUpAttribute]
         public System.Threading.Tasks.Task SetupAsync() { throw null; }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/src/NUnitFrameworkLogger.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+using NUnit.Framework;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit
+{
+    internal class NUnitFrameworkLogger : IFrameworkLogger
+    {
+        public void Debug(string message)
+        {
+            TestContext.WriteLine($"[MPT-NUnit]: {message}");
+        }
+
+        public void Error(string message)
+        {
+            TestContext.Error.WriteLine($"[MPT-NUnit]: {message}");
+        }
+
+        public void Info(string message)
+        {
+            TestContext.Progress.WriteLine($"[MPT-NUnit]: {message}");
+        }
+
+        public void Warning(string message)
+        {
+            TestContext.Progress.WriteLine($"[MPT-NUnit]: {message}");
+        }
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/src/PlaywrightServiceNUnit.cs

@@ -7,6 +7,7 @@
 using System.Runtime.InteropServices;
 using System;
 using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
 
 namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit;
 
@@ -16,12 +17,13 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit;
 [SetUpFixture]
 public class PlaywrightServiceNUnit : PlaywrightService
 {
+    private static NUnitFrameworkLogger nunitFrameworkLogger { get; } = new();
     /// <summary>
     /// Initializes a new instance of the <see cref="PlaywrightServiceNUnit"/> class.
     /// </summary>
     /// <param name="credential">The azure token credential to use for authentication.</param>
     public PlaywrightServiceNUnit(TokenCredential? credential = null)
-        : base(playwrightServiceOptions, credential: credential)
+        : base(playwrightServiceOptions, credential: credential, frameworkLogger: nunitFrameworkLogger)
     {
     }
 
@@ -47,7 +49,7 @@ public async Task SetupAsync()
     {
         if (!UseCloudHostedBrowsers)
             return;
-        TestContext.Progress.WriteLine("\nRunning tests using Microsoft Playwright Testing service.\n");
+        nunitFrameworkLogger.Info("\nRunning tests using Microsoft Playwright Testing service.\n");
 
         await InitializeAsync().ConfigureAwait(false);
     }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs

@@ -8,8 +8,8 @@ public ConnectOptions() { }
     }
     public partial class PlaywrightService
     {
-        public PlaywrightService(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions, Azure.Core.TokenCredential? credential = null) { }
-        public PlaywrightService(System.Runtime.InteropServices.OSPlatform? os = default(System.Runtime.InteropServices.OSPlatform?), string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = default(bool?), Azure.Core.TokenCredential? credential = null) { }
+        public PlaywrightService(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions, Azure.Core.TokenCredential? credential = null, Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger? frameworkLogger = null) { }
+        public PlaywrightService(System.Runtime.InteropServices.OSPlatform? os = default(System.Runtime.InteropServices.OSPlatform?), string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = default(bool?), Azure.Core.TokenCredential? credential = null, Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger? frameworkLogger = null) { }
         public string? ExposeNetwork { get { throw null; } set { } }
         public System.Runtime.InteropServices.OSPlatform? Os { get { throw null; } set { } }
         public System.Threading.Timer? RotationTimer { get { throw null; } set { } }
@@ -67,3 +67,13 @@ public enum ServiceVersion
         }
     }
 }
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface
+{
+    public partial interface IFrameworkLogger
+    {
+        void Debug(string message);
+        void Error(string message);
+        void Info(string message);
+        void Warning(string message);
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/EntraLifecycle.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Core;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
 using Azure.Identity;
 using Microsoft.IdentityModel.JsonWebTokens;
 
@@ -16,9 +17,11 @@ internal class EntraLifecycle
     internal long? _entraIdAccessTokenExpiry;
     private readonly TokenCredential _tokenCredential;
     private readonly JsonWebTokenHandler _jsonWebTokenHandler;
+    private readonly IFrameworkLogger? _frameworkLogger;
 
-    public EntraLifecycle(TokenCredential? tokenCredential = null, JsonWebTokenHandler? jsonWebTokenHandler = null)
+    public EntraLifecycle(TokenCredential? tokenCredential = null, JsonWebTokenHandler? jsonWebTokenHandler = null, IFrameworkLogger? frameworkLogger = null)
     {
+        _frameworkLogger = frameworkLogger;
         _tokenCredential = tokenCredential ?? new DefaultAzureCredential();
         _jsonWebTokenHandler = jsonWebTokenHandler ?? new JsonWebTokenHandler();
         SetEntraIdAccessTokenFromEnvironment();
@@ -37,7 +40,7 @@ internal async Task FetchEntraIdAccessTokenAsync(CancellationToken cancellationT
         }
         catch (Exception ex)
         {
-            Console.Error.WriteLine(ex);
+            _frameworkLogger?.Error(ex.ToString());
             throw new Exception(Constants.s_no_auth_error);
         }
     }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/VSTestFrameworkLogger.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation
+{
+    internal class VSTestFrameworkLogger : IFrameworkLogger
+    {
+        private readonly ILogger _logger;
+        public VSTestFrameworkLogger(ILogger? logger = null)
+        {
+            _logger = logger ?? new Logger();
+        }
+
+        public void Debug(string message)
+        {
+            _logger.Debug(message);
+        }
+
+        public void Error(string message)
+        {
+            _logger.Error(message);
+        }
+
+        public void Info(string message)
+        {
+            _logger.Info(message);
+        }
+
+        public void Warning(string message)
+        {
+            _logger.Warning(message);
+        }
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IFrameworkLogger.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface
+{
+    /// <summary>
+    /// Sets up logging for the TestLogger package.
+    /// </summary>
+    public interface IFrameworkLogger
+    {
+        /// <summary>
+        /// Log informational message.
+        /// </summary>
+        /// <param name="message"></param>
+        void Info(string message);
+        /// <summary>
+        /// Log debug messages.
+        /// </summary>
+        /// <param name="message"></param>
+        void Debug(string message);
+        /// <summary>
+        /// Log warnming messages.
+        /// </summary>
+        /// <param name="message"></param>
+        void Warning(string message);
+        /// <summary>
+        /// Log error messages.
+        /// </summary>
+        /// <param name="message"></param>
+        void Error(string message);
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs

@@ -121,10 +121,19 @@ internal void InitializePlaywrightReporter(string xmlSettings)
             return;
         }
         // setup entra rotation handlers
-        _playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, entraLifecycle: null, jsonWebTokenHandler: _jsonWebTokenHandler, credential: playwrightServiceSettings.AzureTokenCredential);
+        IFrameworkLogger frameworkLogger = new VSTestFrameworkLogger(_logger);
+        try
+        {
+            _playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, entraLifecycle: null, jsonWebTokenHandler: _jsonWebTokenHandler, credential: playwrightServiceSettings.AzureTokenCredential, frameworkLogger: frameworkLogger);
 #pragma warning disable AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead.
-        _playwrightService.InitializeAsync().GetAwaiter().GetResult();
+            _playwrightService.InitializeAsync().GetAwaiter().GetResult();
 #pragma warning restore AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead.
+        }
+        catch (Exception ex)
+        {
+            // We have checks for access token and base url in the next block, so we can ignore the exception here.
+            _logger.Error("Failed to initialize PlaywrightService: " + ex);
+        }
 
         var cloudRunId = _environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId);
         string baseUrl = _environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_REPORTING_URL);

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightService.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using Azure.Core;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
 using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model;
 using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility;
 using Microsoft.IdentityModel.JsonWebTokens;
@@ -136,19 +137,22 @@ public string? ExposeNetwork
 
     private readonly EntraLifecycle? _entraLifecycle;
     private readonly JsonWebTokenHandler? _jsonWebTokenHandler;
+    private IFrameworkLogger? _frameworkLogger;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="PlaywrightService"/> class.
     /// </summary>
     /// <param name="playwrightServiceOptions"></param>
     /// <param name="credential"></param>
-    public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, TokenCredential? credential = null) : this(
+    /// <param name="frameworkLogger"></param>
+    public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null) : this(
         os: playwrightServiceOptions.Os,
         runId: playwrightServiceOptions.RunId,
         exposeNetwork: playwrightServiceOptions.ExposeNetwork,
         serviceAuth: playwrightServiceOptions.ServiceAuth,
         useCloudHostedBrowsers: playwrightServiceOptions.UseCloudHostedBrowsers,
-        credential: credential ?? playwrightServiceOptions.AzureTokenCredential
+        credential: credential ?? playwrightServiceOptions.AzureTokenCredential,
+        frameworkLogger: frameworkLogger
     )
     {
         // No-op
@@ -163,21 +167,25 @@ public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, Toke
     /// <param name="serviceAuth">The service authentication mechanism.</param>
     /// <param name="useCloudHostedBrowsers">Whether to use cloud-hosted browsers.</param>
     /// <param name="credential">The token credential.</param>
-    public PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, TokenCredential? credential = null)
+    /// <param name="frameworkLogger">Logger</param>
+    public PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null)
     {
         if (string.IsNullOrEmpty(ServiceEndpoint))
             return;
-        _entraLifecycle = new EntraLifecycle(tokenCredential: credential);
+        _frameworkLogger = frameworkLogger;
+        _entraLifecycle = new EntraLifecycle(tokenCredential: credential, frameworkLogger: _frameworkLogger);
         _jsonWebTokenHandler = new JsonWebTokenHandler();
         InitializePlaywrightServiceEnvironmentVariables(GetServiceCompatibleOs(os), runId, exposeNetwork, serviceAuth, useCloudHostedBrowsers);
     }
 
-    internal PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, EntraLifecycle? entraLifecycle = null, JsonWebTokenHandler? jsonWebTokenHandler = null, TokenCredential? credential = null)
+    internal PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, EntraLifecycle? entraLifecycle = null, JsonWebTokenHandler? jsonWebTokenHandler = null, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null)
     {
         if (string.IsNullOrEmpty(ServiceEndpoint))
             return;
+        _frameworkLogger = frameworkLogger;
         _jsonWebTokenHandler = jsonWebTokenHandler ?? new JsonWebTokenHandler();
-        _entraLifecycle = entraLifecycle ?? new EntraLifecycle(credential, _jsonWebTokenHandler);
+        _entraLifecycle = entraLifecycle ?? new EntraLifecycle(credential, _jsonWebTokenHandler, _frameworkLogger);
+        _frameworkLogger = frameworkLogger;
         InitializePlaywrightServiceEnvironmentVariables(GetServiceCompatibleOs(os), runId, exposeNetwork, serviceAuth, useCloudHostedBrowsers);
     }
 
@@ -211,6 +219,7 @@ internal PlaywrightService(OSPlatform? os = null, string? runId = null, string?
         }
         if (string.IsNullOrEmpty(GetAuthToken()))
         {
+            _frameworkLogger?.Error("Access token not found when trying to call GetConnectOptionsAsync.");
             throw new Exception(Constants.s_no_auth_error);
         }
 
@@ -236,20 +245,26 @@ internal PlaywrightService(OSPlatform? os = null, string? runId = null, string?
     public async Task InitializeAsync(CancellationToken cancellationToken = default)
     {
         if (string.IsNullOrEmpty(ServiceEndpoint))
+        {
+            _frameworkLogger?.Info("Exiting initialization as service endpoint is not set.");
             return;
+        }
         if (!UseCloudHostedBrowsers)
         {
             // Since playwright-dotnet checks PLAYWRIGHT_SERVICE_ACCESS_TOKEN and PLAYWRIGHT_SERVICE_URL to be set, remove PLAYWRIGHT_SERVICE_URL so that tests are run locally.
             // If customers use GetConnectOptionsAsync, after setting disableScalableExecution, an error will be thrown.
+            _frameworkLogger?.Info("Disabling scalable execution since UseCloudHostedBrowsers is set to false.");
             Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceUri, null);
             return;
         }
         // If default auth mechanism is Access token and token is available in the environment variable, no need to setup rotation handler
         if (ServiceAuth == ServiceAuthType.AccessToken)
         {
+            _frameworkLogger?.Info("Auth mechanism is Access Token.");
             ValidateMptPAT();
             return;
         }
+        _frameworkLogger?.Info("Auth mechanism is Entra Id.");
         await _entraLifecycle!.FetchEntraIdAccessTokenAsync(cancellationToken).ConfigureAwait(false);
         RotationTimer = new Timer(RotationHandlerAsync, null, TimeSpan.FromMinutes(Constants.s_entra_access_token_rotation_interval_period_in_minutes), TimeSpan.FromMinutes(Constants.s_entra_access_token_rotation_interval_period_in_minutes));
     }
@@ -259,13 +274,15 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default)
     /// </summary>
     public void Cleanup()
     {
+        _frameworkLogger?.Info("Cleaning up Playwright service resources.");
         RotationTimer?.Dispose();
     }
 
     internal async void RotationHandlerAsync(object? _)
     {
         if (_entraLifecycle!.DoesEntraIdAccessTokenRequireRotation())
         {
+            _frameworkLogger?.Info("Rotating Entra Id access token.");
             await _entraLifecycle.FetchEntraIdAccessTokenAsync().ConfigureAwait(false);
         }
     }
@@ -360,20 +377,28 @@ internal static void SetReportingUrlAndWorkspaceId()
 
     private void ValidateMptPAT()
     {
-        string authToken = GetAuthToken()!;
-        if (string.IsNullOrEmpty(authToken))
-            throw new Exception(Constants.s_no_auth_error);
-        JsonWebToken jsonWebToken = _jsonWebTokenHandler!.ReadJsonWebToken(authToken) ?? throw new Exception(Constants.s_invalid_mpt_pat_error);
-        var tokenWorkspaceId = jsonWebToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value;
-        Match match = Regex.Match(ServiceEndpoint, @"wss://(?<region>[\w-]+)\.api\.(?<domain>playwright(?:-test|-int)?\.io|playwright\.microsoft\.com)/accounts/(?<workspaceId>[\w-]+)/");
-        if (!match.Success)
-            throw new Exception(Constants.s_invalid_service_endpoint_error_message);
-        var serviceEndpointWorkspaceId = match.Groups["workspaceId"].Value;
-        if (tokenWorkspaceId != serviceEndpointWorkspaceId)
-            throw new Exception(Constants.s_workspace_mismatch_error);
-        var expiry = (long)(jsonWebToken.ValidTo - new DateTime(1970, 1, 1)).TotalSeconds;
-        if (expiry <= DateTimeOffset.UtcNow.ToUnixTimeSeconds())
-            throw new Exception(Constants.s_expired_mpt_pat_error);
+        try
+        {
+            string authToken = GetAuthToken()!;
+            if (string.IsNullOrEmpty(authToken))
+                throw new Exception(Constants.s_no_auth_error);
+            JsonWebToken jsonWebToken = _jsonWebTokenHandler!.ReadJsonWebToken(authToken) ?? throw new Exception(Constants.s_invalid_mpt_pat_error);
+            var tokenWorkspaceId = jsonWebToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value;
+            Match match = Regex.Match(ServiceEndpoint, @"wss://(?<region>[\w-]+)\.api\.(?<domain>playwright(?:-test|-int)?\.io|playwright\.microsoft\.com)/accounts/(?<workspaceId>[\w-]+)/");
+            if (!match.Success)
+                throw new Exception(Constants.s_invalid_service_endpoint_error_message);
+            var serviceEndpointWorkspaceId = match.Groups["workspaceId"].Value;
+            if (tokenWorkspaceId != serviceEndpointWorkspaceId)
+                throw new Exception(Constants.s_workspace_mismatch_error);
+            var expiry = (long)(jsonWebToken.ValidTo - new DateTime(1970, 1, 1)).TotalSeconds;
+            if (expiry <= DateTimeOffset.UtcNow.ToUnixTimeSeconds())
+                throw new Exception(Constants.s_expired_mpt_pat_error);
+        }
+        catch (Exception ex)
+        {
+            _frameworkLogger?.Error(ex.ToString());
+            throw;
+        }
     }
 
     internal static string? GetServiceCompatibleOs(OSPlatform? oSPlatform)