chore(ci-tests): split tests across multiple machines

.github/workflows/workflow.yml

@@ -3,30 +3,84 @@ on:
   # Ensure GitHub actions are not run twice for same commits
   push:
     branches: [main]
-    tags: ['*']
   pull_request:
     types: [opened, synchronize, reopened]
 jobs:
   build:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 30
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        node-version: ['*']
+    steps:
+      # Sets an output parameter if this is a release PR
+      - name: Check for release
+        id: release-check
+        run: echo "::set-output name=IS_RELEASE::true"
+        if: ${{ startsWith(github.head_ref, 'release-') }}
+      - name: Git checkout
+        uses: actions/checkout@v3
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
+      - name: Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
+      - name: Install dependencies
+        run: npm ci
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
+      - name: Linting
+        run: npm run format:ci
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
+  test:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
     strategy:
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
         node-version: ['*']
         install-command: ['npm ci']
+        machine: ['0', '1', '2', '3']
         include:
           # We test on the oldest supported Node.js version, but only with a
           # single combination (Ubuntu)
           - os: ubuntu-latest
             node-version: '12.20.0'
             install-command: npm ci
+            machine: '0'
+          - os: ubuntu-latest
+            node-version: '12.20.0'
+            install-command: npm ci
+            machine: '1'
+          - os: ubuntu-latest
+            node-version: '12.20.0'
+            install-command: npm ci
+            machine: '2'
+          - os: ubuntu-latest
+            node-version: '12.20.0'
+            install-command: npm ci
+            machine: '3'
           # The buildbot pins Netlify Build's dependencies, like `npm ci`.
           # But other consumers do not, like `npm install`.
           # So we test both.
           - os: ubuntu-latest
             node-version: '*'
             install-command: npm install --no-package-lock
+            machine: '0'
+          - os: ubuntu-latest
+            node-version: '*'
+            install-command: npm install --no-package-lock
+            machine: '1'
+          - os: ubuntu-latest
+            node-version: '*'
+            install-command: npm install --no-package-lock
+            machine: '2'
+          - os: ubuntu-latest
+            node-version: '*'
+            install-command: npm install --no-package-lock
+            machine: '3'
       fail-fast: false
     steps:
       # Sets an output parameter if this is a release PR
@@ -38,23 +92,26 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
       - name: Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
           cache: 'npm'
-          check-latest: true
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
       - name: Install npm@7
         run: npm install -g npm@7
-        if: ${{ matrix.node-version == '12.20.0' }}
+        if: ${{ matrix.node-version == '12.20.0' && !steps.release-check.outputs.IS_RELEASE }}
       - name: Install dependencies
         run: ${{ matrix.install-command }}
-      - name: Linting
-        run: npm run format:ci
-        if: ${{ matrix.node-version == '*' && !steps.release-check.outputs.IS_RELEASE }}
+        if: ${{ !steps.release-check.outputs.IS_RELEASE }}
       - name: Tests
         run: npm run test:ci
         if: ${{ !steps.release-check.outputs.IS_RELEASE }}
+        env:
+          # split tests across multiple machines
+          CI_NODE_INDEX: ${{ matrix.machine }}
+          CI_NODE_TOTAL: 4
       - name: Get test coverage flags
         id: test-coverage-flags
         if: ${{ !steps.release-check.outputs.IS_RELEASE }}

CONTRIBUTING.md

@@ -51,6 +51,11 @@ See our [testing documentation](packages/build/tests/README.md) to learn about o
 
 After submitting the pull request, please make sure the Continuous Integration checks (GitHub actions) are passing.
 
+### Testing in CI
+
+To speed up CI, we load balance the tests across multiple machines. The information required to load balance the tests is stored in `tests-metadata.json`, and later used by our test [runner](ava.config.js#L10).
+To regenerate the data (e.g. when adding a new test file) run `npm test:measure` and commit the changes to GitHub.
+
 ## Releasing
 
 For more details, please refer to the

ava.config.js

@@ -0,0 +1,36 @@
+import fs from 'fs'
+import path from 'path'
+import process from 'process'
+
+// eslint-disable-next-line node/no-unpublished-import
+import { isCI } from 'ci-info'
+
+// `tests-metadata.json` is created by running `npm run test:measure`
+// eslint-disable-next-line node/no-sync
+const testData = JSON.parse(fs.readFileSync('tests-metadata.json'))
+
+const getOrder = (file) => {
+  const fileRelative = path.relative(process.cwd(), file).replace(/\\/g, '/')
+  if (testData[fileRelative]) {
+    return testData[fileRelative].order
+  }
+
+  console.warn(`Missing test metadata for ${fileRelative}`)
+  return Number.MAX_SAFE_INTEGER
+}
+
+const sortTestFiles = (file1, file2) => getOrder(file1) - getOrder(file2)
+
+const config = {
+  files: ['packages/**/tests/*.{cjs,mjs,js}', 'packages/**/tests/**/tests.{cjs,mjs,js}'],
+  verbose: true,
+  timeout: '120s',
+  workerThreads: false,
+  environmentVariables: {
+    FORCE_COLOR: '1',
+  },
+  // we only sort in CI to split efficiently across machines
+  ...(isCI && { sortTestFiles }),
+}
+
+export default config