diff --git a/.github/workflows/end2end-test.yml b/.github/workflows/end2end-test.yml
index 9d1f09fd86cf1c..b99a07a0502f73 100644
--- a/.github/workflows/end2end-test.yml
+++ b/.github/workflows/end2end-test.yml
@@ -144,9 +144,9 @@ jobs:
runs-on: 'ubuntu-24.04'
permissions:
contents: read
+ # Needed to comment the flaky tests report on the pull request.
+ pull-requests: write
timeout-minutes: 5
- outputs:
- has-flaky-test-report: ${{ !!steps.merge-flaky-tests-reports.outputs.artifact-id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -192,41 +192,20 @@ jobs:
path: playwright-report
if-no-files-found: ignore
- report-to-issues:
- name: Report to GitHub
- needs: [merge-artifacts]
- if: ${{ needs.merge-artifacts.outputs.has-flaky-test-report == 'true' }}
- runs-on: 'ubuntu-24.04'
- permissions:
- contents: read
- issues: write
- pull-requests: write
- timeout-minutes: 10
- steps:
- # Checkout defaults to using the branch which triggered the event, which
- # isn't necessarily `trunk` (e.g. in the case of a merge).
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- ref: trunk
- show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
- persist-credentials: false
-
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- # Don't fail the job if there isn't any flaky tests report.
- continue-on-error: true
+ # Flaky tests are only reported as a comment on the pull request. The
+ # merge step above only sets `artifact-id` when a report was produced,
+ # so both steps are skipped on runs without any flaky test.
+ - name: Download merged flaky tests report
+ id: download-flaky-tests-report
+ if: ${{ github.event_name == 'pull_request' && steps.merge-flaky-tests-reports.outputs.artifact-id }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: flaky-tests-report
path: flaky-tests
- - name: Setup Node.js and install dependencies
- uses: ./.github/setup-node
-
- - name: Npm build
- run: npm run build -- --skip-types
-
- name: Report flaky tests
+ if: ${{ steps.download-flaky-tests-report.outcome == 'success' }}
uses: ./packages/report-flaky-tests
with:
repo-token: '${{ secrets.GITHUB_TOKEN }}'
- label: '[Type] Flaky Test'
artifact-path: flaky-tests
diff --git a/.github/workflows/stale-issue-gardening.yml b/.github/workflows/stale-issue-gardening.yml
index 660fc8605df47b..86ae038bf005b3 100644
--- a/.github/workflows/stale-issue-gardening.yml
+++ b/.github/workflows/stale-issue-gardening.yml
@@ -33,13 +33,6 @@ jobs:
only-labels: 'Needs Testing'
remove-stale-when-updated: true
stale-issue-label: '[Status] Stale'
- - name: 'Flaky test issues without activity'
- message: 'This issue has gone 15 days without any activity.'
- days-before-stale: 15
- days-before-close: 1
- only-labels: '[Type] Flaky Test'
- remove-stale-when-updated: true
- stale-issue-label: '[Status] Stale'
steps:
- name: Update issues
diff --git a/docs/contributors/code/testing-overview.md b/docs/contributors/code/testing-overview.md
index 237fa32930dc44..e271eedbed6950 100644
--- a/docs/contributors/code/testing-overview.md
+++ b/docs/contributors/code/testing-overview.md
@@ -532,7 +532,7 @@ Every core block is required to have at least one set of fixture files for its m
### Flaky tests
-A test is considered to be **flaky** when it can pass and fail across multiple retry attempts without any code changes. We auto retry failed tests at most **twice** on CI to detect and report them to GitHub issues automatically under the [`[Type] Flaky Test`](https://github.com/WordPress/gutenberg/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22%5BType%5D+Flaky+Test%22) label via [`report-flaky-tests`](https://github.com/WordPress/gutenberg/tree/trunk/packages/report-flaky-tests) GitHub action. Note that a test that failed three times in a row is not counted as a flaky test and will not be reported to an issue.
+A test is considered to be **flaky** when it can pass and fail across multiple retry attempts without any code changes. We auto retry failed tests at most **twice** on CI to detect them, and report them, together with their errors, as a single comment on the pull request via the [`report-flaky-tests`](https://github.com/WordPress/gutenberg/tree/trunk/packages/report-flaky-tests) GitHub action. Note that a test that failed three times in a row is not counted as a flaky test and will not be reported, and that flaky tests are only reported on pull requests.
## PHP testing
diff --git a/package-lock.json b/package-lock.json
index 95a90f00ae44c3..5b11039af9edb5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -53337,7 +53337,6 @@
"@actions/core": "^1.9.1",
"@actions/github": "^5.0.1",
"@jest/test-result": "^30.4.1",
- "@octokit/types": "^6.34.0",
"@octokit/webhooks-types": "^5.8.0",
"@playwright/test": "^1.62.1",
"jest-message-util": "^30.4.1"
diff --git a/packages/report-flaky-tests/CHANGELOG.md b/packages/report-flaky-tests/CHANGELOG.md
index 34c83ae9061ba8..36c5154d2c0b7a 100644
--- a/packages/report-flaky-tests/CHANGELOG.md
+++ b/packages/report-flaky-tests/CHANGELOG.md
@@ -2,4 +2,9 @@
## Unreleased
+### Breaking Changes
+
+- Flaky tests are no longer reported to GitHub issues, and are no longer reported on pushes. The action now only comments the detected tests, together with their errors, on the pull request. The `label` input has been removed as a result.
+- The action now runs straight from its TypeScript sources on Node's type stripping, instead of from a bundle. Relative imports must carry their `.ts` extension, and the syntax must be erasable — no enums, namespaces, parameter properties or decorators.
+
-- Initial version of the package.
diff --git a/packages/report-flaky-tests/README.md b/packages/report-flaky-tests/README.md
index 12b5b06192147c..79e60c9b532237 100644
--- a/packages/report-flaky-tests/README.md
+++ b/packages/report-flaky-tests/README.md
@@ -1,6 +1,8 @@
# Report Flaky Tests
-A GitHub action to report flaky E2E tests to GitHub issues.
+A GitHub action to report flaky E2E tests as a comment on the pull request.
+
+It runs straight from the TypeScript sources in `src/`: the `node24` runtime strips the types on the fly, so there is no build step. Two constraints come with that — relative imports must carry their `.ts` extension, and the syntax must be erasable, which rules out enums, namespaces, parameter properties and decorators.
**This package is still experimental and breaking changes could be introduced in future minor versions (`v0.x`). Use it at your own risks.**
diff --git a/packages/report-flaky-tests/action.yml b/packages/report-flaky-tests/action.yml
index 8685760a28dfe2..4bdc16ad08690a 100644
--- a/packages/report-flaky-tests/action.yml
+++ b/packages/report-flaky-tests/action.yml
@@ -1,17 +1,17 @@
name: 'Report flaky tests'
-description: 'Report flaky tests to GitHub issues'
+description: 'Report flaky tests as a comment on the pull request'
inputs:
repo-token:
description: 'GitHub token'
required: true
- label:
- description: 'The flaky-test label name'
- required: true
- default: 'flaky-test'
artifact-path:
description: 'The path of the downloaded artifact'
required: true
default: 'flaky-tests'
runs:
+ # Node 24 strips the types on the fly, so the action runs straight from its
+ # TypeScript sources without a build step. This requires every relative
+ # import to carry its `.ts` extension, and the syntax to be erasable, which
+ # rules out enums, namespaces, parameter properties and decorators.
using: 'node24'
- main: 'build/index.cjs'
+ main: 'src/index.ts'
diff --git a/packages/report-flaky-tests/package.json b/packages/report-flaky-tests/package.json
index 3d9892a176d80a..3613806d8e2180 100644
--- a/packages/report-flaky-tests/package.json
+++ b/packages/report-flaky-tests/package.json
@@ -2,7 +2,7 @@
"name": "@wordpress/report-flaky-tests",
"version": "0.0.0",
"private": true,
- "description": "A GitHub action to report flaky E2E tests to GitHub issues.",
+ "description": "A GitHub action to report flaky E2E tests as a comment on the pull request.",
"author": "The WordPress Contributors",
"license": "GPL-2.0-or-later",
"keywords": [
@@ -23,11 +23,12 @@
"node": ">=18.12.0",
"npm": ">=8.19.2"
},
- "main": "build/index.mjs",
+ "type": "module",
+ "main": "src/index.ts",
"exports": {
".": {
"types": "./build-types/index.d.ts",
- "require": "./build/index.mjs"
+ "import": "./src/index.ts"
},
"./package.json": "./package.json"
},
@@ -36,7 +37,6 @@
"@actions/core": "^1.9.1",
"@actions/github": "^5.0.1",
"@jest/test-result": "^30.4.1",
- "@octokit/types": "^6.34.0",
"@octokit/webhooks-types": "^5.8.0",
"@playwright/test": "^1.62.1",
"jest-message-util": "^30.4.1"
diff --git a/packages/report-flaky-tests/src/__fixtures__/flaky-issues.json b/packages/report-flaky-tests/src/__fixtures__/flaky-issues.json
deleted file mode 100644
index 749fddb2ae7e60..00000000000000
--- a/packages/report-flaky-tests/src/__fixtures__/flaky-issues.json
+++ /dev/null
@@ -1,189 +0,0 @@
-[
- {
- "url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303",
- "repository_url": "https://api.github.com/repos/WordPress/gutenberg",
- "labels_url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303/labels{/name}",
- "comments_url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303/comments",
- "events_url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303/events",
- "html_url": "https://github.com/WordPress/gutenberg/issues/40303",
- "id": 1203058634,
- "node_id": "I_kwDOBNHdeM5HtTfK",
- "number": 40303,
- "title": "[Flaky Test] should copy only partial selection of text blocks",
- "user": {
- "login": "github-actions[bot]",
- "id": 41898282,
- "node_id": "MDM6Qm90NDE4OTgyODI=",
- "avatar_url": "https://avatars.githubusercontent.com/in/15368?v=4",
- "gravatar_id": "",
- "url": "https://api.github.com/users/github-actions%5Bbot%5D",
- "html_url": "https://github.com/apps/github-actions",
- "followers_url": "https://api.github.com/users/github-actions%5Bbot%5D/followers",
- "following_url": "https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}",
- "gists_url": "https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}",
- "starred_url": "https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}",
- "subscriptions_url": "https://api.github.com/users/github-actions%5Bbot%5D/subscriptions",
- "organizations_url": "https://api.github.com/users/github-actions%5Bbot%5D/orgs",
- "repos_url": "https://api.github.com/users/github-actions%5Bbot%5D/repos",
- "events_url": "https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}",
- "received_events_url": "https://api.github.com/users/github-actions%5Bbot%5D/received_events",
- "type": "Bot",
- "site_admin": false
- },
- "labels": [
- {
- "id": 587343997,
- "node_id": "MDU6TGFiZWw1ODczNDM5OTc=",
- "url": "https://api.github.com/repos/WordPress/gutenberg/labels/[Status]%20In%20Progress",
- "name": "[Status] In Progress",
- "color": "0052cc",
- "default": false,
- "description": "Tracking issues with work in progress"
- },
- {
- "id": 3337407158,
- "node_id": "MDU6TGFiZWwzMzM3NDA3MTU4",
- "url": "https://api.github.com/repos/WordPress/gutenberg/labels/[Type]%20Flaky%20Test",
- "name": "[Type] Flaky Test",
- "color": "B6860B",
- "default": false,
- "description": "Auto-generated flaky test report issue"
- }
- ],
- "state": "open",
- "locked": false,
- "assignee": {
- "login": "ntsekouras",
- "id": 16275880,
- "node_id": "MDQ6VXNlcjE2Mjc1ODgw",
- "avatar_url": "https://avatars.githubusercontent.com/u/16275880?v=4",
- "gravatar_id": "",
- "url": "https://api.github.com/users/ntsekouras",
- "html_url": "https://github.com/ntsekouras",
- "followers_url": "https://api.github.com/users/ntsekouras/followers",
- "following_url": "https://api.github.com/users/ntsekouras/following{/other_user}",
- "gists_url": "https://api.github.com/users/ntsekouras/gists{/gist_id}",
- "starred_url": "https://api.github.com/users/ntsekouras/starred{/owner}{/repo}",
- "subscriptions_url": "https://api.github.com/users/ntsekouras/subscriptions",
- "organizations_url": "https://api.github.com/users/ntsekouras/orgs",
- "repos_url": "https://api.github.com/users/ntsekouras/repos",
- "events_url": "https://api.github.com/users/ntsekouras/events{/privacy}",
- "received_events_url": "https://api.github.com/users/ntsekouras/received_events",
- "type": "User",
- "site_admin": false
- },
- "assignees": [
- {
- "login": "ntsekouras",
- "id": 16275880,
- "node_id": "MDQ6VXNlcjE2Mjc1ODgw",
- "avatar_url": "https://avatars.githubusercontent.com/u/16275880?v=4",
- "gravatar_id": "",
- "url": "https://api.github.com/users/ntsekouras",
- "html_url": "https://github.com/ntsekouras",
- "followers_url": "https://api.github.com/users/ntsekouras/followers",
- "following_url": "https://api.github.com/users/ntsekouras/following{/other_user}",
- "gists_url": "https://api.github.com/users/ntsekouras/gists{/gist_id}",
- "starred_url": "https://api.github.com/users/ntsekouras/starred{/owner}{/repo}",
- "subscriptions_url": "https://api.github.com/users/ntsekouras/subscriptions",
- "organizations_url": "https://api.github.com/users/ntsekouras/orgs",
- "repos_url": "https://api.github.com/users/ntsekouras/repos",
- "events_url": "https://api.github.com/users/ntsekouras/events{/privacy}",
- "received_events_url": "https://api.github.com/users/ntsekouras/received_events",
- "type": "User",
- "site_admin": false
- }
- ],
- "milestone": null,
- "comments": 6,
- "created_at": "2022-04-13T09:45:37Z",
- "updated_at": "2022-05-17T12:57:58Z",
- "closed_at": null,
- "author_association": "NONE",
- "active_lock_reason": null,
- "body": "\n**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**\n\n## Test title\nshould copy only partial selection of text blocks\n\n## Test path\n`/home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js`\n\n## Flaky rate (_estimated_)\n`58 / 192` runs\n\n## Errors\n\n\r\n\r\n\r\n\t[2022-04-13T10:32:59.201Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:218:52\r\n```\r\n \r\n...\r\n\r\n\r\n\t[2022-04-18T09:51:38.785Z]update/post-featured-image-component \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T10:53:30.411Z]add/wp-env/wp-env-core-env \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T12:20:51.800Z]fix/e2e-test \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T13:47:15.612Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T13:48:06.669Z]fix/duotone-site-editor \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T14:01:10.151Z]fix/playwright-snapshots \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T14:12:20.129Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T14:48:36.649Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T14:56:37.960Z]rnmobile/feature/drag-and-drop \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:02:27.183Z]rnmobile/feature/drag-and-drop-prevent-text-focus-on-long-press \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:34:59.990Z]add/section-concept \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:38:30.538Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:48:40.494Z]rnmobile/feature/drag-and-drop-refactor-draggable \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:50:50.420Z]rnmobile/feature/drag-and-drop-update-chip-animation \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:52:48.948Z]fix/comment-reply-link-alignment \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T15:55:31.386Z]rnmobile/feature/drag-and-drop-haptic-feedback \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T16:09:33.323Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T16:17:20.282Z]fix/embed-block-preview-cut-off \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T17:30:31.287Z]trunk \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T17:51:21.537Z]fix/copy-from-non-text-inputs \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T19:54:17.115Z]remove/block-styles-remove-role \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T21:02:44.582Z]docgen/replace-fixtures-with-code \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-18T22:38:25.644Z]try/use-css-var-for-user-presets \r\n\r\n```\r\nError: undefinedSnapshot comparison failed:undefined\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A undefinedundefinedundefinedblockundefinedundefinedundefinedundefined
undefined\r\n undefinedundefined\r\n\r\n undefinedundefined\r\n undefinedundefined
\r\n \r\n\r\n \r\n undefinedundefinedundefinedB undefinedundefinedundefinedblock
\r\n \r\n\r\nExpected: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txtundefined\r\nReceived: undefined/home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txtundefined\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-04-23T16:41:35.814Z]fix/flaky-test-reporter \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n...\r\n\r\n\r\n\t[2022-05-10T13:20:09.628Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-10T13:33:14.282Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-10T13:40:06.465Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-10T15:45:55.917Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-10T15:54:10.994Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-10T18:58:47.148Z]fix/input-field-reset-behavior-moar \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-16T07:23:49.752Z]refactor/range-control-to-typescript \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-16T07:50:53.486Z]refactor/range-control-to-typescript \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\t[2022-05-16T09:51:42.310Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\r\n\r\n\r\n\t[2022-05-16T10:08:17.403Z]wp/6.0 \r\n\r\n```\r\nError: Snapshot comparison failed:\r\n\r\n \r\n block
\r\n \r\n\r\n \r\n B
\r\n \r\n\r\n \r\n A block
\r\n \r\n\r\n \r\n
\r\n \r\n\r\n \r\n B block
\r\n \r\n\r\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\r\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\r\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\r\n```\r\n \r\n\n\n\n\t[2022-05-17T12:57:57.490Z]wp/6.0 \n\n```\nError: Snapshot comparison failed:\n\n \n block
\n \n\n \n B
\n \n\n \n A block
\n \n\n \n
\n \n\n \n B block
\n \n\nExpected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt\nReceived: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt\n at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52\n```\n \n\n",
- "reactions": {
- "url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303/reactions",
- "total_count": 0,
- "+1": 0,
- "-1": 0,
- "laugh": 0,
- "hooray": 0,
- "confused": 0,
- "heart": 0,
- "rocket": 0,
- "eyes": 0
- },
- "timeline_url": "https://api.github.com/repos/WordPress/gutenberg/issues/40303/timeline",
- "performed_via_github_app": null
- },
- {
- "url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763",
- "repository_url": "https://api.github.com/repos/WordPress/gutenberg",
- "labels_url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763/labels{/name}",
- "comments_url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763/comments",
- "events_url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763/events",
- "html_url": "https://github.com/WordPress/gutenberg/issues/39763",
- "id": 1180871796,
- "node_id": "I_kwDOBNHdeM5GYqx0",
- "number": 39763,
- "title": "[Flaky Test] should handle Enter across blocks",
- "user": {
- "login": "github-actions[bot]",
- "id": 41898282,
- "node_id": "MDM6Qm90NDE4OTgyODI=",
- "avatar_url": "https://avatars.githubusercontent.com/in/15368?v=4",
- "gravatar_id": "",
- "url": "https://api.github.com/users/github-actions%5Bbot%5D",
- "html_url": "https://github.com/apps/github-actions",
- "followers_url": "https://api.github.com/users/github-actions%5Bbot%5D/followers",
- "following_url": "https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}",
- "gists_url": "https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}",
- "starred_url": "https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}",
- "subscriptions_url": "https://api.github.com/users/github-actions%5Bbot%5D/subscriptions",
- "organizations_url": "https://api.github.com/users/github-actions%5Bbot%5D/orgs",
- "repos_url": "https://api.github.com/users/github-actions%5Bbot%5D/repos",
- "events_url": "https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}",
- "received_events_url": "https://api.github.com/users/github-actions%5Bbot%5D/received_events",
- "type": "Bot",
- "site_admin": false
- },
- "labels": [
- {
- "id": 3337407158,
- "node_id": "MDU6TGFiZWwzMzM3NDA3MTU4",
- "url": "https://api.github.com/repos/WordPress/gutenberg/labels/[Type]%20Flaky%20Test",
- "name": "[Type] Flaky Test",
- "color": "B6860B",
- "default": false,
- "description": "Auto-generated flaky test report issue"
- }
- ],
- "state": "open",
- "locked": false,
- "assignee": null,
- "assignees": [],
- "milestone": null,
- "comments": 0,
- "created_at": "2022-03-25T14:36:30Z",
- "updated_at": "2022-03-29T13:42:56Z",
- "closed_at": null,
- "author_association": "NONE",
- "active_lock_reason": null,
- "body": "\n**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**\n\n## Test title\nshould handle Enter across blocks\n\n## Test path\n`specs/editor/various/multi-block-selection.test.js`\n\n## Flaky rate (_estimated_)\n`0 / 0` runs\n\n## Errors\n\n\n\t[2022-03-29T13:42:56.215Z]fix/use-entity-records-eslint-warning \n\n```\n ● Multi-block selection › should handle Enter across blocks\n\n expect(received).toMatchSnapshot()\n\n Snapshot name: `Multi-block selection should handle Enter across blocks 2`\n\n - Snapshot - 1\n + Received + 5\n\n @@ -1,7 +1,11 @@\n \n - 1
\n + 1[
\n + \n +\n + \n + .
\n \n\n \n &
\n \n\n 884 |\n 885 | \t\t// Expect two blocks with \"&\" in between.\n > 886 | \t\texpect( await getEditedPostContent() ).toMatchSnapshot();\n | \t\t ^\n 887 | \t} );\n 888 |\n 889 | \tit( 'should merge into quote with correct selection', async () => {\n\n at Object. (specs/editor/various/multi-block-selection.test.js:886:42)\n at runMicrotasks ()\n\n```\n \n",
- "reactions": {
- "url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763/reactions",
- "total_count": 0,
- "+1": 0,
- "-1": 0,
- "laugh": 0,
- "hooray": 0,
- "confused": 0,
- "heart": 0,
- "rocket": 0,
- "eyes": 0
- },
- "timeline_url": "https://api.github.com/repos/WordPress/gutenberg/issues/39763/timeline",
- "performed_via_github_app": null
- }
-]
diff --git a/packages/report-flaky-tests/src/__tests__/__snapshots__/markdown.test.ts.snap b/packages/report-flaky-tests/src/__tests__/__snapshots__/markdown.test.ts.snap
index 0764b3d4435faa..eb3c6a5932a1ec 100644
--- a/packages/report-flaky-tests/src/__tests__/__snapshots__/markdown.test.ts.snap
+++ b/packages/report-flaky-tests/src/__tests__/__snapshots__/markdown.test.ts.snap
@@ -68,36 +68,3 @@ exports[`formatTestErrorMessage should format test error message for jest-circus
at runMicrotasks ()
"
`;
-
-exports[`parseIssueBody should parse issue body 1`] = `
-"
-**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
-
-## Test title
-Test title
-
-## Test path
-\`test/e2e/specs/test-title.spec.js\`
-
-## Errors
-
-[2020-08-01T00:00:00.000Z]trunk
-
- [2020-05-10T00:00:00.000Z]try/some-branch
-
-\`\`\`
-Error:
- Oops...
- Oops again
-\`\`\`
-
-
-Some broken log.
-Oops.
-
-
-
-"
-`;
diff --git a/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap b/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap
index a94e1c18175e50..caff3f69dc7654 100644
--- a/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap
+++ b/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap
@@ -1,93 +1,14 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
-exports[`Report flaky tests should report flaky tests to issue on pull request: Updated existing flaky issue 1`] = `
-"
-**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
+exports[`Report flaky tests should comment on the pull request 1`] = `
+"
+**Flaky tests detected in headSHA.**
+Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
-## Test title
-should copy only partial selection of text blocks
-
-## Test path
-\`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\`
-
-## Errors
-
-[2022-04-13T10:32:59.201Z]trunk[2022-04-18T09:51:38.785Z]update/post-featured-image-component[2022-04-18T10:53:30.411Z]add/wp-env/wp-env-core-env[2022-04-18T12:20:51.800Z]fix/e2e-test[2022-04-18T13:47:15.612Z]trunk[2022-04-18T13:48:06.669Z]fix/duotone-site-editor[2022-04-18T14:01:10.151Z]fix/playwright-snapshots[2022-04-18T14:12:20.129Z]trunk[2022-04-18T14:48:36.649Z]trunk[2022-04-18T14:56:37.960Z]rnmobile/feature/drag-and-drop[2022-04-18T15:02:27.183Z]rnmobile/feature/drag-and-drop-prevent-text-focus-on-long-press[2022-04-18T15:34:59.990Z]add/section-concept[2022-04-18T15:38:30.538Z]trunk[2022-04-18T15:48:40.494Z]rnmobile/feature/drag-and-drop-refactor-draggable[2022-04-18T15:50:50.420Z]rnmobile/feature/drag-and-drop-update-chip-animation[2022-04-18T15:52:48.948Z]fix/comment-reply-link-alignment[2022-04-18T15:55:31.386Z]rnmobile/feature/drag-and-drop-haptic-feedback[2022-04-18T16:09:33.323Z]trunk[2022-04-18T16:17:20.282Z]fix/embed-block-preview-cut-off[2022-04-18T17:30:31.287Z]trunk[2022-04-18T17:51:21.537Z]fix/copy-from-non-text-inputs[2022-04-18T19:54:17.115Z]remove/block-styles-remove-role[2022-04-18T21:02:44.582Z]docgen/replace-fixtures-with-code[2022-04-18T22:38:25.644Z]try/use-css-var-for-user-presets[2022-04-23T16:41:35.814Z]fix/flaky-test-reporter[2022-05-10T13:20:09.628Z]wp/6.0[2022-05-10T13:33:14.282Z]wp/6.0[2022-05-10T13:40:06.465Z]wp/6.0[2022-05-10T15:45:55.917Z]wp/6.0[2022-05-10T15:54:10.994Z]wp/6.0[2022-05-10T18:58:47.148Z]fix/input-field-reset-behavior-moar[2022-05-16T07:23:49.752Z]refactor/range-control-to-typescript[2022-05-16T07:50:53.486Z]refactor/range-control-to-typescript[2022-05-16T09:51:42.310Z]wp/6.0[2022-05-16T10:08:17.403Z]wp/6.0[2022-05-17T12:57:57.490Z]wp/6.0
-
- [2020-05-10T00:00:00.000Z]headBranch
+🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100
+📝 Reported tests:
+
+should copy only partial selection of text blocks in /test/e2e/specs/editor/various/copy-cut-paste.spec.js, passed after 1 failed attempt.
\`\`\`
Error: Snapshot comparison failed:
@@ -116,27 +37,10 @@ Expected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-va
Received: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt
at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52
\`\`\`
-
-
-"
-`;
-
-exports[`Report flaky tests should report flaky tests to issue on push: Created new flaky issue 1`] = `
-"
-**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
-
-## Test title
-Should insert new template part on creation
-## Test path
-\`specs/site-editor/template-part.test.js\`
-
-## Errors
-
-
-
- [2020-05-10T00:00:00.000Z]trunk
+
+
+Should insert new template part on creation in specs/site-editor/template-part.test.js, passed after 2 failed attempts.
\`\`\`
● Template Part › Template part block › Template part placeholder › Should insert new template part on creation
@@ -178,128 +82,6 @@ Should insert new template part on creation
at runMicrotasks ()
\`\`\`
-
-
-"
-`;
-
-exports[`Report flaky tests should report flaky tests to issue on push: Updated existing flaky issue 1`] = `
-"
-**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
-
-## Test title
-should copy only partial selection of text blocks
-
-## Test path
-\`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\`
-
-## Errors
-
-[2022-04-13T10:32:59.201Z]trunk[2022-04-18T09:51:38.785Z]update/post-featured-image-component[2022-04-18T10:53:30.411Z]add/wp-env/wp-env-core-env[2022-04-18T12:20:51.800Z]fix/e2e-test[2022-04-18T13:47:15.612Z]trunk[2022-04-18T13:48:06.669Z]fix/duotone-site-editor[2022-04-18T14:01:10.151Z]fix/playwright-snapshots[2022-04-18T14:12:20.129Z]trunk[2022-04-18T14:48:36.649Z]trunk[2022-04-18T14:56:37.960Z]rnmobile/feature/drag-and-drop[2022-04-18T15:02:27.183Z]rnmobile/feature/drag-and-drop-prevent-text-focus-on-long-press[2022-04-18T15:34:59.990Z]add/section-concept[2022-04-18T15:38:30.538Z]trunk[2022-04-18T15:48:40.494Z]rnmobile/feature/drag-and-drop-refactor-draggable[2022-04-18T15:50:50.420Z]rnmobile/feature/drag-and-drop-update-chip-animation[2022-04-18T15:52:48.948Z]fix/comment-reply-link-alignment[2022-04-18T15:55:31.386Z]rnmobile/feature/drag-and-drop-haptic-feedback[2022-04-18T16:09:33.323Z]trunk[2022-04-18T16:17:20.282Z]fix/embed-block-preview-cut-off[2022-04-18T17:30:31.287Z]trunk[2022-04-18T17:51:21.537Z]fix/copy-from-non-text-inputs[2022-04-18T19:54:17.115Z]remove/block-styles-remove-role[2022-04-18T21:02:44.582Z]docgen/replace-fixtures-with-code[2022-04-18T22:38:25.644Z]try/use-css-var-for-user-presets[2022-04-23T16:41:35.814Z]fix/flaky-test-reporter[2022-05-10T13:20:09.628Z]wp/6.0[2022-05-10T13:33:14.282Z]wp/6.0[2022-05-10T13:40:06.465Z]wp/6.0[2022-05-10T15:45:55.917Z]wp/6.0[2022-05-10T15:54:10.994Z]wp/6.0[2022-05-10T18:58:47.148Z]fix/input-field-reset-behavior-moar[2022-05-16T07:23:49.752Z]refactor/range-control-to-typescript[2022-05-16T07:50:53.486Z]refactor/range-control-to-typescript[2022-05-16T09:51:42.310Z]wp/6.0[2022-05-16T10:08:17.403Z]wp/6.0[2022-05-17T12:57:57.490Z]wp/6.0
-
- [2020-05-10T00:00:00.000Z]trunk
-
-\`\`\`
-Error: Snapshot comparison failed:
-
-
- block
-
-
-
- B
-
-
-
- A block
-
-
-
-
-
-
- B block
-
-
-Expected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt
-Received: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt
- at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52
-\`\`\`
-
-
-"
+ "
`;
diff --git a/packages/report-flaky-tests/src/__tests__/markdown.test.ts b/packages/report-flaky-tests/src/__tests__/markdown.test.ts
index 30de48de610f1f..64d60e8a0405ab 100644
--- a/packages/report-flaky-tests/src/__tests__/markdown.test.ts
+++ b/packages/report-flaky-tests/src/__tests__/markdown.test.ts
@@ -1,25 +1,12 @@
-/**
- * External dependencies
- */
-import * as core from '@actions/core';
-
/**
* Internal dependencies
*/
import {
formatTestErrorMessage,
- formatTestResults,
- renderIssueBody,
- parseFormattedTestResults,
- parseIssueBody,
- renderCommitComment,
+ renderReportComment,
isReportComment,
} from '../markdown';
-import type { ReportedIssue } from '../types';
-
-jest.useFakeTimers( 'modern' ).setSystemTime( new Date( '2020-05-10' ) );
-
-jest.spyOn( core, 'error' ).mockImplementation( () => {} );
+import type { ReportedFlakyTest } from '../types';
describe( 'formatTestErrorMessage', () => {
it( 'should format test error message for jest-circus', async () => {
@@ -43,213 +30,26 @@ describe( 'formatTestErrorMessage', () => {
} );
} );
-describe( 'formatTestResults', () => {
- it( 'should format flaky test results without error messages', () => {
- const formatted = formatTestResults( {
- date: new Date(),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2282393879',
- } );
-
- expect( formatted ).toMatchInlineSnapshot(
- `"[2020-05-10T00:00:00.000Z]trunk
-
- [2020-05-10T00:00:00.000Z]trunk
-
- \`\`\`
- Error:
- Oops...
-
- \`\`\`
- "
- ` );
- } );
-} );
-
-describe( 'renderIssueBody', () => {
- it( 'should render issue body', () => {
- const meta = {
- failedTimes: 5,
- totalCommits: 100,
- };
-
- const view = renderIssueBody( {
- meta,
- testTitle: 'Test title',
- testPath: 'test/e2e/specs/test-title.spec.js',
- formattedTestResults: 'FORMATTED_TEST_RESULTS',
- } );
-
- expect( view ).toMatchInlineSnapshot( `
- "
- **Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
-
- ## Test title
- Test title
-
- ## Test path
- \`test/e2e/specs/test-title.spec.js\`
-
- ## Errors
-
- FORMATTED_TEST_RESULTS
-
- "
- ` );
- } );
-} );
-
-describe( 'parseFormattedTestResults', () => {
- it( 'should parse formatted test results without error messages', () => {
- const testResult = {
- date: new Date(),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2297863316',
- };
-
- const formatted = formatTestResults( testResult );
-
- const parsed = parseFormattedTestResults( formatted );
-
- expect( parsed ).toEqual( testResult );
- } );
-
- it( 'should parse formatted test results with error messages', () => {
- const testResult = {
- date: new Date(),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2297863316',
- errorMessage: 'Error:\n Oops...\n',
- };
-
- const formatted = formatTestResults( testResult );
-
- const parsed = parseFormattedTestResults( formatted );
-
- expect( parsed ).toEqual( testResult );
- } );
-
- it( 'should handle GitHub converted line breaks (\r\n) in the result', () => {
- const testResult = {
- date: new Date(),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2297863316',
- errorMessage: 'Error:\n Oops...\n',
- };
-
- const formatted = formatTestResults( testResult ).replace(
- /\n/g,
- '\r\n'
- );
-
- const parsed = parseFormattedTestResults( formatted );
-
- expect( parsed ).toEqual( testResult );
- } );
-} );
-
-describe( 'parseIssueBody', () => {
- it( 'should parse issue body', async () => {
- const testResults = [
- formatTestResults( {
- date: new Date( '2020-08-01' ),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2282393879',
- } ),
- formatTestResults( {
- date: new Date(),
- failedTimes: 2,
- headBranch: 'try/some-branch',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2297863316',
- errorMessage: 'Error:\n Oops...\n Oops again',
- } ),
- `\nSome broken log.\nOops.\n\n`,
- ];
-
- const meta = {
- failedTimes: 5,
- totalCommits: 95,
- };
-
- const view = renderIssueBody( {
- meta,
- testTitle: 'Test title',
- testPath: 'test/e2e/specs/test-title.spec.js',
- formattedTestResults: testResults.join( '\n' ),
- } );
-
- expect( view ).toMatchSnapshot();
-
- const parsed = parseIssueBody( view );
-
- expect( core.error ).toHaveBeenCalledTimes( 1 );
-
- expect( parsed ).toEqual( {
- meta,
- testResults: [
- {
- date: new Date( '2020-08-01' ),
- failedTimes: 1,
- headBranch: 'trunk',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2282393879',
- },
- {
- date: new Date(),
- failedTimes: 2,
- headBranch: 'try/some-branch',
- runURL: 'https://github.com/WordPress/gutenberg/actions/runs/2297863316',
- errorMessage: 'Error:\n Oops...\n Oops again',
- },
- ],
- } );
- } );
-} );
-
-describe( 'renderCommitComment', () => {
- it( 'render the commit comment', () => {
+describe( 'renderReportComment', () => {
+ it( 'render the report comment', () => {
const runURL = 'runURL';
- const reportedIssues: ReportedIssue[] = [
+ const reportedTests: ReportedFlakyTest[] = [
{
testTitle: 'title1',
testPath: 'path1',
- issueNumber: 1,
- issueUrl: 'url1',
+ failedTimes: 1,
+ errorMessage: 'Error:\n Oops...\n',
},
{
testTitle: 'title2',
testPath: 'path2',
- issueNumber: 2,
- issueUrl: 'url2',
+ failedTimes: 2,
},
];
const commitSHA = 'commitSHA';
- const view = renderCommitComment( {
- reportedIssues,
+ const view = renderReportComment( {
+ reportedTests,
runURL,
commitSHA,
} );
@@ -260,17 +60,26 @@ describe( 'renderCommitComment', () => {
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: runURL
- 📝 Reported issues:
- - #1 in \`path1\`
- - #2 in \`path2\`"
+ 📝 Reported tests:
+
+ title1 in path1, passed after 1 failed attempt.
+
+ \`\`\`
+ Error:
+ Oops...
+
+ \`\`\`
+
+
+ title2 in path2, passed after 2 failed attempts.
"
` );
} );
} );
describe( 'isReportComment', () => {
it( 'matches the report comment', () => {
- const view = renderCommitComment( {
- reportedIssues: [],
+ const view = renderReportComment( {
+ reportedTests: [],
runURL: '',
commitSHA: 'commitSHA',
} );
@@ -280,9 +89,3 @@ describe( 'isReportComment', () => {
expect( isReportComment( 'random string' ) ).toBe( false );
} );
} );
-
-function renderToDisplayText( html: string ) {
- const container = document.createElement( 'div' );
- container.innerHTML = html;
- return container.textContent!.replace( /\s+/g, ' ' );
-}
diff --git a/packages/report-flaky-tests/src/__tests__/run.test.ts b/packages/report-flaky-tests/src/__tests__/run.test.ts
index f194d59c04fee6..299b8e34942bd8 100644
--- a/packages/report-flaky-tests/src/__tests__/run.test.ts
+++ b/packages/report-flaky-tests/src/__tests__/run.test.ts
@@ -8,8 +8,6 @@ import * as core from '@actions/core';
*/
import { run } from '../run';
-jest.useFakeTimers( 'modern' ).setSystemTime( new Date( '2020-05-10' ) );
-
const mockPushEventContext = {
runId: 100,
repo: {
@@ -56,12 +54,7 @@ jest.mock( '@actions/core', () => ( {
} ) );
const mockAPI = {
- fetchAllIssuesLabeledFlaky: jest.fn(),
- findMergeBaseCommit: jest.fn(),
- updateIssue: jest.fn(),
- createIssue: jest.fn(),
createCommentOnPR: jest.fn(),
- createCommentOnCommit: jest.fn(),
};
jest.mock( '../github-api', () => ( {
GitHubAPI: jest.fn( () => mockAPI ),
@@ -72,64 +65,49 @@ jest.mock( 'fs/promises', () => ( {
readFile: jest.fn(),
} ) );
+async function mockFlakyTestsArtifact() {
+ const playwrightFlakyTest = await import(
+ '../__fixtures__/should copy only partial selection of text blocks.json'
+ ).then( ( json ) => json.default );
+ const jestFlakyTest = await import(
+ '../__fixtures__/Should insert new template part on creation.json'
+ ).then( ( json ) => json.default );
+
+ ( core.getInput as jest.Mock )
+ // token
+ .mockReturnValueOnce( 'repo-token' )
+ // artifact-path
+ .mockReturnValueOnce( 'flaky-tests' );
+
+ // Replacing the cwd for the test for consistent snapshot results.
+ playwrightFlakyTest.path = playwrightFlakyTest.path.replace(
+ '/home/runner/work/gutenberg/gutenberg',
+ process.cwd()
+ );
+
+ const mockedFs = require( 'fs/promises' );
+ mockedFs.readdir.mockImplementationOnce( () =>
+ Promise.resolve( [
+ `${ playwrightFlakyTest.title }.json`,
+ `${ jestFlakyTest.title }.json`,
+ ] )
+ );
+ mockedFs.readFile
+ .mockImplementationOnce( () =>
+ Promise.resolve( JSON.stringify( playwrightFlakyTest ) )
+ )
+ .mockImplementationOnce( () =>
+ Promise.resolve( JSON.stringify( jestFlakyTest ) )
+ );
+}
+
describe( 'Report flaky tests', () => {
afterEach( () => {
jest.clearAllMocks();
} );
- it( 'should report flaky tests to issue on pull request', async () => {
- const existingFlakyTest = await import(
- '../__fixtures__/should copy only partial selection of text blocks.json'
- ).then( ( json ) => json.default );
- const newFlakyTest = await import(
- '../__fixtures__/Should insert new template part on creation.json'
- ).then( ( json ) => json.default );
- const flakyIssues = await import(
- '../__fixtures__/flaky-issues.json'
- ).then( ( json ) => json.default );
-
- ( core.getInput as jest.Mock )
- // token
- .mockReturnValueOnce( 'repo-token' )
- // artifact-path
- .mockReturnValueOnce( 'flaky-tests' )
- // label
- .mockReturnValueOnce( '[Type] Flaky Test' );
-
- // Replacing the cwd for the test for consistent snapshot results.
- existingFlakyTest.path = existingFlakyTest.path.replace(
- '/home/runner/work/gutenberg/gutenberg',
- process.cwd()
- );
-
- const mockedFs = require( 'fs/promises' );
- mockedFs.readdir.mockImplementationOnce( () =>
- Promise.resolve( [
- `${ existingFlakyTest.title }.json`,
- `${ newFlakyTest.title }.json`,
- ] )
- );
- mockedFs.readFile
- .mockImplementationOnce( () =>
- Promise.resolve( JSON.stringify( existingFlakyTest ) )
- )
- .mockImplementationOnce( () =>
- Promise.resolve( JSON.stringify( newFlakyTest ) )
- );
-
- mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce(
- () => flakyIssues
- );
-
- mockAPI.updateIssue.mockImplementationOnce( () => ( {
- number: 1,
- html_url: 'html_url',
- } ) );
-
- mockAPI.createIssue.mockImplementationOnce( () => ( {
- number: 2,
- html_url: 'html_url',
- } ) );
+ it( 'should comment on the pull request', async () => {
+ await mockFlakyTestsArtifact();
mockAPI.createCommentOnPR.mockImplementationOnce( () => ( {
html_url: 'comment_html_url',
@@ -137,194 +115,35 @@ describe( 'Report flaky tests', () => {
await run();
- const existingFlakyIssue = flakyIssues.find(
- ( issue ) =>
- issue.title === `[Flaky Test] ${ existingFlakyTest.title }`
- )!;
- expect( mockAPI.updateIssue ).toHaveBeenCalledWith(
- expect.objectContaining( {
- issue_number: existingFlakyIssue.number,
- state: 'open',
- } )
- );
- expect( mockAPI.updateIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot(
- 'Updated existing flaky issue'
- );
-
- expect( mockAPI.createIssue ).not.toHaveBeenCalled();
-
expect( mockAPI.createCommentOnPR ).toHaveBeenCalledTimes( 1 );
expect( mockAPI.createCommentOnPR.mock.calls[ 0 ][ 0 ] ).toBe( 10 );
- expect( mockAPI.createCommentOnPR.mock.calls[ 0 ][ 1 ] )
- .toMatchInlineSnapshot( `
- "
- **Flaky tests detected in headSHA.**
- Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
-
- 🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100
- 📝 Reported issues:
- - #1 in \`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\`"
- ` );
+ expect(
+ mockAPI.createCommentOnPR.mock.calls[ 0 ][ 1 ]
+ ).toMatchSnapshot();
} );
- it( 'should report flaky tests to issue on push', async () => {
+ it( 'should skip events other than pull requests', async () => {
mockGetContext.mockImplementation( () => mockPushEventContext );
- const existingFlakyTest = await import(
- '../__fixtures__/should copy only partial selection of text blocks.json'
- ).then( ( json ) => json.default );
- const newFlakyTest = await import(
- '../__fixtures__/Should insert new template part on creation.json'
- ).then( ( json ) => json.default );
- const flakyIssues = await import(
- '../__fixtures__/flaky-issues.json'
- ).then( ( json ) => json.default );
-
- ( core.getInput as jest.Mock )
- // token
- .mockReturnValueOnce( 'repo-token' )
- // artifact-path
- .mockReturnValueOnce( 'flaky-tests' )
- // label
- .mockReturnValueOnce( '[Type] Flaky Test' );
-
- // Replacing the cwd for the test for consistent snapshot results.
- existingFlakyTest.path = existingFlakyTest.path.replace(
- '/home/runner/work/gutenberg/gutenberg',
- process.cwd()
- );
-
- const mockedFs = require( 'fs/promises' );
- mockedFs.readdir.mockImplementationOnce( () =>
- Promise.resolve( [
- `${ existingFlakyTest.title }.json`,
- `${ newFlakyTest.title }.json`,
- ] )
- );
- mockedFs.readFile
- .mockImplementationOnce( () =>
- Promise.resolve( JSON.stringify( existingFlakyTest ) )
- )
- .mockImplementationOnce( () =>
- Promise.resolve( JSON.stringify( newFlakyTest ) )
- );
-
- mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce(
- () => flakyIssues
- );
-
- mockAPI.updateIssue.mockImplementationOnce( () => ( {
- number: 1,
- html_url: 'html_url',
- } ) );
-
- mockAPI.createIssue.mockImplementationOnce( () => ( {
- number: 2,
- html_url: 'html_url',
- } ) );
-
- mockAPI.createCommentOnCommit.mockImplementationOnce( () => ( {
- html_url: 'comment_html_url',
- } ) );
-
await run();
- const existingFlakyIssue = flakyIssues.find(
- ( issue ) =>
- issue.title === `[Flaky Test] ${ existingFlakyTest.title }`
- )!;
- expect( mockAPI.updateIssue ).toHaveBeenCalledWith(
- expect.objectContaining( {
- issue_number: existingFlakyIssue.number,
- state: 'open',
- } )
- );
- expect( mockAPI.updateIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot(
- 'Updated existing flaky issue'
- );
-
- expect( mockAPI.createIssue ).toHaveBeenCalledWith(
- expect.objectContaining( {
- title: `[Flaky Test] ${ newFlakyTest.title }`,
- } )
- );
- expect( mockAPI.createIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot(
- 'Created new flaky issue'
- );
-
- expect( mockAPI.createCommentOnCommit ).toHaveBeenCalledTimes( 1 );
- expect( mockAPI.createCommentOnCommit.mock.calls[ 0 ][ 0 ] ).toBe(
- 'commitSHA'
- );
- expect( mockAPI.createCommentOnCommit.mock.calls[ 0 ][ 1 ] )
- .toMatchInlineSnapshot( `
- "
- **Flaky tests detected in commitSHA.**
- Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
-
- 🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100
- 📝 Reported issues:
- - #1 in \`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\`
- - #2 in \`specs/site-editor/template-part.test.js\`"
- ` );
+ // It bails out before even reading the artifact.
+ expect( require( 'fs/promises' ).readdir ).not.toHaveBeenCalled();
+ expect( mockAPI.createCommentOnPR ).not.toHaveBeenCalled();
mockGetContext.mockImplementation( () => mockPullRequestEventContext );
} );
- it( 'should skip for outdated branches', async () => {
- const flakyTest = await import(
- '../__fixtures__/should copy only partial selection of text blocks.json'
- ).then( ( json ) => json.default );
- const flakyIssues = await import(
- '../__fixtures__/flaky-issues.json'
- ).then( ( json ) => json.default );
-
+ it( 'should not comment when there are no flaky tests', async () => {
( core.getInput as jest.Mock )
.mockReturnValueOnce( 'repo-token' )
- .mockReturnValueOnce( 'flaky-tests-report' )
- .mockReturnValueOnce( '[Type] Flaky Test' );
-
- // Replacing the cwd for the test for consistent snapshot results.
- flakyTest.path = flakyTest.path.replace(
- '/home/runner/work/gutenberg/gutenberg',
- process.cwd()
- );
+ .mockReturnValueOnce( 'flaky-tests' );
const mockedFs = require( 'fs/promises' );
- mockedFs.readdir.mockImplementationOnce( () =>
- Promise.resolve( [ `${ flakyTest.title }.json` ] )
- );
- mockedFs.readFile.mockImplementationOnce( () =>
- Promise.resolve( JSON.stringify( flakyTest ) )
- );
-
- mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce(
- () => flakyIssues
- );
-
- const latestAncestorDate = new Date( '2022-05-10' );
- mockAPI.findMergeBaseCommit.mockImplementationOnce( () => ( {
- committer: {
- date: latestAncestorDate.toISOString(),
- },
- } ) );
-
- // The issue is closed after the latest ancestor date,
- // indicating that the code base at this point is outdated.
- const flakyIssue = flakyIssues.find(
- ( issue ) => issue.title === `[Flaky Test] ${ flakyTest.title }`
- )!;
- flakyIssue.state = 'closed';
- // @ts-expect-error: "closed_at" hasn't been typed yet.
- flakyIssue.closed_at = new Date( '2022-05-15' ).toISOString();
+ mockedFs.readdir.mockImplementationOnce( () => Promise.resolve( [] ) );
await run();
- expect( mockAPI.findMergeBaseCommit ).toHaveBeenCalledTimes( 1 );
-
- expect( mockAPI.updateIssue ).not.toHaveBeenCalled();
-
expect( mockAPI.createCommentOnPR ).not.toHaveBeenCalled();
- expect( mockAPI.createCommentOnCommit ).not.toHaveBeenCalled();
} );
} );
diff --git a/packages/report-flaky-tests/src/github-api.ts b/packages/report-flaky-tests/src/github-api.ts
index c4e52d229c612b..511ecd36491ead 100644
--- a/packages/report-flaky-tests/src/github-api.ts
+++ b/packages/report-flaky-tests/src/github-api.ts
@@ -3,7 +3,6 @@
*/
import { getOctokit } from '@actions/github';
import type { GitHub } from '@actions/github/lib/utils';
-import type { Endpoints } from '@octokit/types';
type Octokit = InstanceType< typeof GitHub >;
@@ -21,93 +20,6 @@ class GitHubAPI {
this.#repo = repo;
}
- async fetchAllIssuesLabeledFlaky( label: string ) {
- const issues = await this.#octokit.paginate(
- this.#octokit.rest.issues.listForRepo,
- {
- ...this.#repo,
- state: 'all',
- labels: label,
- }
- );
-
- return issues;
- }
-
- async findMergeBaseCommit( baseCommit: string, headCommit: string ) {
- const { data } = await this.#octokit.rest.repos.compareCommits( {
- ...this.#repo,
- base: baseCommit,
- head: headCommit,
- per_page: 1,
- } );
-
- return data.merge_base_commit.commit;
- }
-
- async updateIssue(
- params: Omit<
- Endpoints[ 'PATCH /repos/{owner}/{repo}/issues/{issue_number}' ][ 'parameters' ],
- keyof Repo
- >
- ) {
- const { data } = await this.#octokit.rest.issues.update( {
- ...this.#repo,
- ...params,
- } );
-
- return data;
- }
-
- async createIssue(
- params: Omit<
- Endpoints[ 'POST /repos/{owner}/{repo}/issues' ][ 'parameters' ],
- keyof Repo
- >
- ) {
- const { data } = await this.#octokit.rest.issues.create( {
- ...this.#repo,
- ...params,
- } );
-
- return data;
- }
-
- async createCommentOnCommit(
- sha: string,
- body: string,
- isReportComment: ( body: string ) => boolean
- ) {
- const { data: comments } =
- await this.#octokit.rest.repos.listCommentsForCommit( {
- ...this.#repo,
- commit_sha: sha,
- } );
- const reportComment = comments.find( ( comment ) =>
- isReportComment( comment.body )
- );
-
- if ( reportComment ) {
- const { data } = await this.#octokit.rest.repos.updateCommitComment(
- {
- ...this.#repo,
- comment_id: reportComment.id,
- body,
- }
- );
-
- return data;
- }
-
- const { data } = await this.#octokit.rest.repos.createCommitComment( {
- ...this.#repo,
- commit_sha: sha,
- body,
- } );
-
- return data;
- }
-
async createCommentOnPR(
prNumber: number,
body: string,
diff --git a/packages/report-flaky-tests/src/index.ts b/packages/report-flaky-tests/src/index.ts
index ecfd1bb6bf8066..8beca57148f539 100644
--- a/packages/report-flaky-tests/src/index.ts
+++ b/packages/report-flaky-tests/src/index.ts
@@ -6,7 +6,7 @@ import * as core from '@actions/core';
/**
* Internal dependencies
*/
-import { run } from './run';
+import { run } from './run.ts';
run().catch( ( error ) => {
core.error( error instanceof Error ? error : String( error ) );
diff --git a/packages/report-flaky-tests/src/markdown.ts b/packages/report-flaky-tests/src/markdown.ts
index b92fdaca5a799d..ff34a6e4651d12 100644
--- a/packages/report-flaky-tests/src/markdown.ts
+++ b/packages/report-flaky-tests/src/markdown.ts
@@ -3,84 +3,12 @@
*/
import path from 'path';
import { formatResultsErrors } from 'jest-message-util';
-import * as core from '@actions/core';
/**
* Internal dependencies
*/
-import { stripAnsi } from './strip-ansi';
-import type { MetaData, FlakyTestResult, ReportedIssue } from './types';
-
-type ParsedTestResult = {
- date: Date;
- failedTimes: number;
- runURL: string;
- headBranch: string;
- errorMessage?: string;
-};
-
-const TEST_RESULTS_LIST = {
- open: ``,
- close: ``,
-};
-const TEST_RESULT = {
- open: '',
- close: '',
-};
-
-const metaData = {
- render: ( json: MetaData ) =>
- ``,
- get: ( str: string ): MetaData | undefined => {
- const matched = str.match( // );
- try {
- if ( matched ) {
- return JSON.parse( matched[ 1 ] );
- }
- } catch {
- // Ignore errors.
- }
- return undefined;
- },
-};
-
-const TEST_LOG_REGEX = new RegExp(
- `.* \\s*Test passed after (?\\d) failed attempts? on (?.+) \\s*\\s*)?${ TEST_LOG_REGEX.source }(?:\\s* \\s*
-\`\`\`
-(?[\\s\\S]*)
-\`\`\`
-\\s* )?`
-);
-
-function renderIssueBody( {
- meta,
- testTitle,
- testPath,
- formattedTestResults,
-}: {
- meta: MetaData;
- testTitle: string;
- testPath: string;
- formattedTestResults: string;
-} ) {
- return `${ metaData.render( meta ) }
-**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.**
-
-## Test title
-${ testTitle }
-
-## Test path
-\`${ testPath }\`
-
-## Errors
-${ TEST_RESULTS_LIST.open }
-${ formattedTestResults }
-${ TEST_RESULTS_LIST.close }
-`;
-}
+import { stripAnsi } from './strip-ansi.ts';
+import type { FlakyTestResult, ReportedFlakyTest } from './types.ts';
function formatTestErrorMessage( flakyTestResult: FlakyTestResult ) {
switch ( flakyTestResult.runner ) {
@@ -113,104 +41,38 @@ function formatTestErrorMessage( flakyTestResult: FlakyTestResult ) {
}
}
-function formatTestResults( {
- date,
+const FLAKY_TESTS_REPORT_COMMENT_TOKEN = `flaky-tests-report-comment`;
+
+function renderReportedTest( {
+ testTitle,
+ testPath,
failedTimes,
- headBranch,
- runURL,
errorMessage,
-}: ParsedTestResult ) {
- const dateString = date.toISOString();
-
- // It will look something like this without formatting:
- // [2021-08-31T16:15:19.875Z] Test passed after 2 failed attempts on trunk.
- const log = `[${ dateString }]${ testPath }, passed after ${ failedTimes } failed ${
failedTimes === 1 ? 'attempt' : 'attempts'
- } on ${ headBranch }${ summary }
`;
}
- return `${ TEST_RESULT.open }
-
- ${ log }
-
+ return `
+${ summary }
\`\`\`
${ errorMessage }
\`\`\`
- ${ TEST_RESULT.close }`;
-}
-
-function parseFormattedTestResults(
- formattedTestResults: string
-): ParsedTestResult {
- const matches = formattedTestResults
- // Unify line breaks.
- .replace( /\r\n/g, '\n' )
- .match( TEST_FULL_REGEX );
-
- if ( ! matches ) {
- throw new Error( `Unable to parse the test results:
-${ formattedTestResults }` );
- }
- const { date, failedTimes, runURL, headBranch, errorMessage } =
- matches.groups!;
-
- return {
- date: new Date( date ),
- failedTimes: parseInt( failedTimes, 10 ),
- runURL,
- headBranch,
- errorMessage,
- };
+ `;
}
-function parseIssueBody( body: string ) {
- const meta = metaData.get( body );
-
- if ( ! meta ) {
- throw new Error( 'Unable to parse meta data from issue.' );
- }
-
- const testResults = Array.from(
- body.matchAll(
- new RegExp(
- `^${ TEST_RESULT.open }[\\s\\S]+?${ TEST_RESULT.close }$`,
- 'gm'
- )
- )
- )
- .map( ( [ match ] ) => match )
- .map( ( testResult ) => {
- try {
- return parseFormattedTestResults( testResult );
- } catch ( error ) {
- core.error( error as Error );
- return undefined;
- }
- } )
- .filter(
- ( testResult ): testResult is ParsedTestResult =>
- testResult !== undefined
- );
-
- return {
- meta,
- testResults,
- };
-}
-
-const FLAKY_TESTS_REPORT_COMMENT_TOKEN = `flaky-tests-report-comment`;
-
-function renderCommitComment( {
- reportedIssues,
+function renderReportComment( {
+ reportedTests,
runURL,
commitSHA,
}: {
- reportedIssues: ReportedIssue[];
+ reportedTests: ReportedFlakyTest[];
runURL: string;
commitSHA: string;
} ) {
@@ -219,22 +81,12 @@ function renderCommitComment( {
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: ${ runURL }
-📝 Reported issues:
-${ reportedIssues
- .map( ( issue ) => `- #${ issue.issueNumber } in \`${ issue.testPath }\`` )
- .join( '\n' ) }`;
+📝 Reported tests:
+${ reportedTests.map( renderReportedTest ).join( '\n' ) }`;
}
function isReportComment( body: string ) {
return body.startsWith( `` );
}
-export {
- renderIssueBody,
- formatTestErrorMessage,
- formatTestResults,
- parseFormattedTestResults,
- parseIssueBody,
- renderCommitComment,
- isReportComment,
-};
+export { formatTestErrorMessage, renderReportComment, isReportComment };
diff --git a/packages/report-flaky-tests/src/run.ts b/packages/report-flaky-tests/src/run.ts
index d68ced5be24037..39c9d8bb9ba6c4 100644
--- a/packages/report-flaky-tests/src/run.ts
+++ b/packages/report-flaky-tests/src/run.ts
@@ -1,38 +1,46 @@
/**
* External dependencies
*/
-import * as github from '@actions/github';
-import * as core from '@actions/core';
import * as fs from 'fs/promises';
import * as path from 'path';
+import * as github from '@actions/github';
+import * as core from '@actions/core';
import type { PullRequestEvent } from '@octokit/webhooks-types';
/**
* Internal dependencies
*/
-import { GitHubAPI } from './github-api';
+import { GitHubAPI } from './github-api.ts';
import {
- renderIssueBody,
formatTestErrorMessage,
- formatTestResults,
- parseIssueBody,
- renderCommitComment,
+ renderReportComment,
isReportComment,
-} from './markdown';
-import type { ReportedIssue } from './types';
+} from './markdown.ts';
+import type { FlakyTestResult, ReportedFlakyTest } from './types.ts';
async function run() {
+ // Flaky tests are only reported on pull requests. The workflow already
+ // filters other events out, so this is a safety net for reuse elsewhere.
+ if ( github.context.eventName !== 'pull_request' ) {
+ core.info(
+ `Flaky tests are only reported on pull requests, skipping the "${ github.context.eventName }" event.`
+ );
+ return;
+ }
+
const token = core.getInput( 'repo-token', { required: true } );
const artifactPath = core.getInput( 'artifact-path', {
required: true,
} );
- const { runId: runID, repo, ref } = github.context;
+ const { runId: runID, repo } = github.context;
+ // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions
+ const payload = github.context.payload as PullRequestEvent;
const runURL = `https://github.com/${ repo.owner }/${ repo.repo }/actions/runs/${ runID }`;
const api = new GitHubAPI( token, repo );
const flakyTestsDir = await fs.readdir( artifactPath );
- const flakyTests = await Promise.all(
+ const flakyTests: FlakyTestResult[] = await Promise.all(
flakyTestsDir.map( ( filename ) =>
fs
.readFile( path.join( artifactPath, filename ), 'utf-8' )
@@ -45,157 +53,28 @@ async function run() {
return;
}
- const isPR = github.context.eventName === 'pull_request';
- const headBranch = isPR
- ? // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions
- ( github.context.payload as PullRequestEvent ).pull_request.head.ref
- : ref.replace( /^refs\/(heads|tag)\//, '' );
-
- const label = core.getInput( 'label', { required: true } );
- const issues = await api.fetchAllIssuesLabeledFlaky( label );
- const reportedIssues: ReportedIssue[] = [];
-
- for ( const flakyTest of flakyTests ) {
- const { title: testTitle } = flakyTest;
- const issueTitle = getIssueTitle( testTitle );
- const reportedIssue = issues.find(
- ( issue ) => issue.title === issueTitle
- );
- const testPath = flakyTest.path.startsWith( process.cwd() )
- ? flakyTest.path.slice( process.cwd().length )
- : flakyTest.path;
- let issue;
-
- const currentFormattedTestResults = formatTestResults( {
- date: new Date(),
+ const reportedTests: ReportedFlakyTest[] = flakyTests.map(
+ ( flakyTest ) => ( {
+ testTitle: flakyTest.title,
+ testPath: flakyTest.path.startsWith( process.cwd() )
+ ? flakyTest.path.slice( process.cwd().length )
+ : flakyTest.path,
failedTimes: flakyTest.results.length,
- headBranch,
- runURL,
- // Always output the latest test results' stacks.
errorMessage: formatTestErrorMessage( flakyTest ),
- } );
-
- if ( reportedIssue ) {
- const body = reportedIssue.body!;
-
- // The issue is closed.
- if ( reportedIssue.closed_at ) {
- try {
- // Represent the latest base commit on trunk.
- const latestAncestorCommit = await api.findMergeBaseCommit(
- 'trunk',
- github.context.sha
- );
- const latestAncestorDate =
- latestAncestorCommit.committer?.date;
-
- // Cannot find the commit date, skipping.
- if ( ! latestAncestorDate ) {
- return;
- }
-
- // The issue is closed after the latest base commit on trunk,
- // which means the branch/PR/commit is outdated and the flaky test
- // has probably already been fixed. Skip reporting the outdated flaky tests.
- if (
- new Date( reportedIssue.closed_at ) >=
- new Date( latestAncestorDate )
- ) {
- return;
- }
- } catch ( error ) {
- // It might be a deleted commit or something else.
- core.error(
- error instanceof Error ? error : String( error )
- );
- return;
- }
- }
-
- const { meta, testResults: prevTestResults } =
- parseIssueBody( body );
-
- // Concat the test results list with the latest test results.
- const formattedTestResults = [
- ...prevTestResults.map( ( testResult ) =>
- formatTestResults( {
- ...testResult,
- // Don't output previous test results' stacks.
- errorMessage: undefined,
- } )
- ),
- currentFormattedTestResults,
- ].join( '\n