From 531e819401e92b8c6df0dd8e11d47078e4d5b647 Mon Sep 17 00:00:00 2001 From: Khafra Date: Thu, 9 May 2024 19:22:05 -0400 Subject: [PATCH 1/2] fail wpt runner if expected failures does not match actual --- test/wpt/runner/runner.mjs | 21 +++++++++++++++++++-- test/wpt/runner/util.mjs | 2 +- test/wpt/status/FileAPI.status.json | 3 ++- test/wpt/status/fetch.status.json | 3 ++- test/wpt/status/websockets.status.json | 12 ------------ 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs index 33c1b757bbf..0a4a3c26767 100644 --- a/test/wpt/runner/runner.mjs +++ b/test/wpt/runner/runner.mjs @@ -218,7 +218,7 @@ export class WPTRunner extends EventEmitter { if (message.type === 'result') { this.handleIndividualTestCompletion(message, status, test, meta, result) } else if (message.type === 'completion') { - this.handleTestCompletion(worker) + this.handleTestCompletion(worker, status, test) } else if (message.type === 'error') { this.#uncaughtExceptions.push({ error: message.error, test }) this.#stats.failedTests += 1 @@ -318,8 +318,25 @@ export class WPTRunner extends EventEmitter { * Called after all the tests in a worker are completed. * @param {Worker} worker */ - handleTestCompletion (worker) { + handleTestCompletion (worker, status, path) { worker.terminate() + + const { file } = status + const hasExpectedFailures = !!file.fail + const testHasFailures = !!this.#statusOutput?.[path] + + if (hasExpectedFailures !== testHasFailures) { + const failed = this.#statusOutput?.[path] ?? [] + console.log({ expected: file.fail, failed }) + + if (failed.length === 0) { + console.log(colors('Tests are marked as failure but did not fail, yay!', 'red')) + } else if (!hasExpectedFailures) { + console.log(colors('Test failed but there were no expected errors.', 'red')) + } + + process.exitCode = 1 + } } /** diff --git a/test/wpt/runner/util.mjs b/test/wpt/runner/util.mjs index ec284df68b2..6bf8b777123 100644 --- a/test/wpt/runner/util.mjs +++ b/test/wpt/runner/util.mjs @@ -168,5 +168,5 @@ export function resolveStatusPath (path, status) { } } - return { topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} } + return { fullPath: path, topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} } } diff --git a/test/wpt/status/FileAPI.status.json b/test/wpt/status/FileAPI.status.json index c64d255a0d3..da7aa96c0c8 100644 --- a/test/wpt/status/FileAPI.status.json +++ b/test/wpt/status/FileAPI.status.json @@ -11,7 +11,8 @@ "skip": true }, "Blob-stream.any.js": { - "fail": [ + "note": "fails in node v18", + "flaky": [ "Reading Blob.stream() with BYOB reader" ] } diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index c94ef2f1c3b..45ae1fca46c 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -394,7 +394,8 @@ ] }, "response-consume-stream.any.js": { - "fail": [ + "note": "only fail in node v18", + "flaky": [ "Read blob response's body as readableStream with mode=byob", "Read text response's body as readableStream with mode=byob", "Read URLSearchParams response's body as readableStream with mode=byob", diff --git a/test/wpt/status/websockets.status.json b/test/wpt/status/websockets.status.json index 9f5a6308a1d..edd27727820 100644 --- a/test/wpt/status/websockets.status.json +++ b/test/wpt/status/websockets.status.json @@ -16,12 +16,6 @@ } } }, - "Create-blocked-port.any.js": { - "note": "TODO(@KhafraDev): investigate failure", - "fail": [ - "Basic check" - ] - }, "Send-binary-arraybufferview-float32.any.js": { "note": "TODO(@KhafraDev): investigate failure", "fail": [ @@ -58,12 +52,6 @@ "Send binary data on a WebSocket - ArrayBufferView - Uint32Array with offset - Connection should be closed" ] }, - "basic-auth.any.js": { - "note": "TODO(@KhafraDev): investigate failure", - "fail": [ - "HTTP basic authentication should work with WebSockets" - ] - }, "Create-on-worker-shutdown.any.js": { "skip": true, "//": "Node.js workers are different from web workers & don't work with blob: urls" From fb91e5d744a08fdad9872bfc7d8745551c90a341 Mon Sep 17 00:00:00 2001 From: Khafra Date: Thu, 9 May 2024 19:43:12 -0400 Subject: [PATCH 2/2] fixup --- test/wpt/runner/runner.mjs | 13 ++++++++++++- test/wpt/status/fetch.status.json | 11 ++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs index 0a4a3c26767..3d79125f6a9 100644 --- a/test/wpt/runner/runner.mjs +++ b/test/wpt/runner/runner.mjs @@ -324,9 +324,9 @@ export class WPTRunner extends EventEmitter { const { file } = status const hasExpectedFailures = !!file.fail const testHasFailures = !!this.#statusOutput?.[path] + const failed = this.#statusOutput?.[path] ?? [] if (hasExpectedFailures !== testHasFailures) { - const failed = this.#statusOutput?.[path] ?? [] console.log({ expected: file.fail, failed }) if (failed.length === 0) { @@ -336,6 +336,17 @@ export class WPTRunner extends EventEmitter { } process.exitCode = 1 + } else if (hasExpectedFailures && testHasFailures) { + const diff = [ + ...file.fail.filter(x => !failed.includes(x)), + ...failed.filter(x => !file.fail.includes(x)) + ] + + if (diff.length) { + console.log({ diff }) + console.log(colors('Expected failures did not match actual failures', 'red')) + process.exitCode = 1 + } } } diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index 45ae1fca46c..9d63c1df11c 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -3,10 +3,7 @@ "abort": { "general.any.js": { "fail": [ - "Already aborted signal rejects immediately", - "Underlying connection is closed when aborting after receiving response - no-cors", - "Stream errors once aborted. Underlying connection closed.", - "Readable stream synchronously cancels with AbortError if aborted before reading" + "Already aborted signal rejects immediately" ] }, "cache.https.any.js": { @@ -52,9 +49,6 @@ }, "referrer.any.js": { "fail": [ - "origin-when-cross-origin policy on a cross-origin URL", - "origin-when-cross-origin policy on a cross-origin URL after same-origin redirection", - "origin-when-cross-origin policy on a same-origin URL after cross-origin redirection", "origin-when-cross-origin policy on a same-origin URL" ] }, @@ -100,7 +94,7 @@ }, "stream-safe-creation.any.js": { "note": "tests are very finnicky", - "fail": [ + "flaky": [ "throwing Object.prototype.type accessor should not affect stream creation by 'fetch'", "Object.prototype.type accessor returning invalid value should not affect stream creation by 'fetch'", "throwing Object.prototype.highWaterMark accessor should not affect stream creation by 'fetch'", @@ -344,7 +338,6 @@ "Adding invalid request header \"Accept-Encoding: KO\"", "Adding invalid request header \"Access-Control-Request-Headers: KO\"", "Adding invalid request header \"Access-Control-Request-Method: KO\"", - "Adding invalid request header \"Access-Control-Request-Private-Network: KO\"", "Adding invalid request header \"Connection: KO\"", "Adding invalid request header \"Content-Length: KO\"", "Adding invalid request header \"Cookie: KO\"",