Make Playwright throw exceptions for 500s and test

backend/Testing/Browser/Base/PageTest.cs

@@ -17,6 +17,11 @@ public class PageTest : IAsyncLifetime
     public IPage Page => _fixture.Page;
     public IBrowser Browser => _fixture.Browser;
     public IBrowserContext Context => _fixture.Context;
+    /// <summary>
+    /// Exceptions that are deferred until the end of the test, because they can't
+    /// be cleanly thrown in sub-threads.
+    /// </summary>
+    private List<UnexpectedResponseException> DeferredExceptions { get; } = new();
 
     public PageTest()
     {
@@ -26,6 +31,21 @@ public PageTest()
     public ILocatorAssertions Expect(ILocator locator) => Assertions.Expect(locator);
     public IPageAssertions Expect(IPage page) => Assertions.Expect(page);
     public IAPIResponseAssertions Expect(IAPIResponse response) => Assertions.Expect(response);
+    /// <summary>
+    /// Consumes a deferred exception that was "thrown" in a sub-thread, and returns it
+    /// or throws if no exception of the given type is found.
+    /// </summary>
+    public UnexpectedResponseException ExpectDeferredException()
+    {
+        var exception = DeferredExceptions.ShouldHaveSingleItem();
+        DeferredExceptions.Clear();
+        return exception;
+    }
+
+    public void ExpectNoDeferredExceptions()
+    {
+        DeferredExceptions.ShouldBeEmpty();
+    }
 
     public virtual async Task InitializeAsync()
     {
@@ -39,6 +59,14 @@ await Context.Tracing.StartAsync(new()
                 Sources = true
             });
         }
+
+        Context.Response += (_, response) =>
+        {
+            if (response.Status >= (int)HttpStatusCode.InternalServerError)
+            {
+                DeferredExceptions.Add(new UnexpectedResponseException(response));
+            }
+        };
     }
 
     public virtual async Task DisposeAsync()
@@ -52,6 +80,11 @@ public virtual async Task DisposeAsync()
         }
 
         await _fixture.DisposeAsync();
+
+        if (DeferredExceptions.Any())
+        {
+            throw new AggregateException(DeferredExceptions);
+        }
     }
 
     static readonly HttpClient HttpClient = new HttpClient();

backend/Testing/Browser/Base/UnexpectedResponseException.cs

@@ -0,0 +1,23 @@
+using System.Text.RegularExpressions;
+using Microsoft.Playwright;
+
+namespace Testing.Browser.Base;
+
+public partial class UnexpectedResponseException : SystemException
+{
+    public static string MaskUrl(string url)
+    {
+        return JwtRegex().Replace(url, "*****");
+    }
+
+    public UnexpectedResponseException(IResponse response)
+    : this(response.StatusText, response.Status, response.Url)
+    {
+    }
+
+    public UnexpectedResponseException(string statusText, int statusCode, string url)
+    : base($"Unexpected response: {statusText} ({statusCode}). URL: {MaskUrl(url)}.") { }
+
+    [GeneratedRegex("[A-Za-z0-9-_]{10,}\\.[A-Za-z0-9-_]{20,}\\.[A-Za-z0-9-_]{10,}")]
+    private static partial Regex JwtRegex();
+}

backend/Testing/Browser/SandboxPageTests.cs

@@ -1,4 +1,4 @@
-using Shouldly;
+using Microsoft.Playwright;
 using Testing.Browser.Base;
 using Testing.Browser.Page;
 
@@ -8,16 +8,51 @@ namespace Testing.Browser;
 public class SandboxPageTests : PageTest
 {
     [Fact]
-    public async Task Goto500Works()
+    public async Task CatchGoto500InSameTab()
+    {
+
+        await new SandboxPage(Page).Goto();
+        await Page.RunAndWaitForResponseAsync(async () =>
+        {
+            await Page.GetByText("Goto 500 page").ClickAsync();
+        }, "/api/testing/test500NoException");
+        ExpectDeferredException();
+    }
+
+    [Fact]
+    public async Task CatchGoto500InNewTab()
     {
         await new SandboxPage(Page).Goto();
-        var request = await Page.RunAndWaitForRequestFinishedAsync(async () =>
+        await Context.RunAndWaitForPageAsync(async () =>
         {
-            await Page.GetByText("goto 500 page").ClickAsync();
+            await Page.GetByText("goto 500 new tab").ClickAsync();
         });
-        var response = await request.ResponseAsync();
-        response.ShouldNotBeNull();
-        response.Ok.ShouldBeFalse();
-        response.Status.ShouldBe(500);
+        ExpectDeferredException();
+    }
+
+    [Fact(Skip = "Playwright doesn't catch the document load request of pages opened with Ctrl+Click")]
+    public async Task CatchGoto500InNewTabWithCtrl()
+    {
+        await new SandboxPage(Page).Goto();
+        await Context.RunAndWaitForPageAsync(async () =>
+        {
+            await Page.GetByText("Goto 500 page").ClickAsync(new()
+            {
+                Modifiers = new[] { KeyboardModifier.Control },
+            });
+        });
+        ExpectDeferredException();
+    }
+
+    [Fact]
+    public async Task CatchFetch500()
+    {
+        await new SandboxPage(Page).Goto();
+        await Page.RunAndWaitForResponseAsync(async () =>
+        {
+            await Page.GetByText("Fetch 500").ClickAsync();
+        }, "/api/testing/test500NoException");
+        ExpectDeferredException();
+        await Expect(Page.Locator(".modal-box.bg-error:has-text('Internal Server Error (500)')")).ToBeVisibleAsync();
     }
 }

frontend/src/routes/(unauthenticated)/sandbox/+page.svelte

@@ -5,11 +5,17 @@
   function uploadFinished(): void {
       alert('upload done!');
   }
+
+  async function fetch500(): Promise<Response> {
+    return fetch('/api/testing/test500NoException');
+  }
 </script>
 <div class="grid gap-2">
   <h2>Sandbox</h2>
   <div class="card w-96 bg-base-200 shadow-lg">
     <a rel="external" class="btn" href="/api/testing/test500NoException">Goto 500 page</a>
+    <a rel="external" target="_blank" class="btn" href="/api/testing/test500NoException">Goto 500 new tab</a>
+    <button class="btn" on:click={fetch500}>Fetch 500</button>
   </div>
   <div class="card w-96 bg-base-200 shadow-lg">
     <div class="card-body">