chore: use Nx affected tasks in CI (twentyhq#5110)

Closes twentyhq#5097 - Uses "nx affected" to detect what projects need to be checked in the current PR (for now, `ci-front` and `ci-server` workflows only). - Caches results of certain tasks (`lint`, `typecheck`, `test`, `storybook:build`) when a PR pipeline runs. The next runs of the same PR's pipeline will then be able to reuse the PR's task cache to execute tasks faster. - Caches Yarn's cache folder to install dependencies faster in CI jobs. - Rewrites the node modules cache/install steps as a custom, reusable Github action. - Distributes `ci-front` jobs with a "matrix" strategy. - Sets common tasks config at the root `nx.json`. For instance, to activate the `typecheck` task in a project, add `typecheck: {}` to its `project.json` and it'll use the default config set in `nx.json` for the `typecheck` task. Options can be overridden in each individual `project.json` if needed. - Adds "scope" tags to some projects: `scope:frontend`, `scope:backend`, `scope:shared`. An eslint rule ensures that `scope:frontend` only depends on `scope:frontent` or `scope:shared` projects, same for `scope:backend`. These tags are used by `nx affected` to filter projects by scope and generates different task cache keys according to the requested scope. - Enables checks for twenty-emails in the `ci-server` workflow.
arnavsaxena17 · Apr 30, 2024 · 11f56bf · 11f56bf
1 parent 7e1de81
commit 11f56bf
Show file tree

Hide file tree

Showing 44 changed files with 13,400 additions and 13,389 deletions.
diff --git a/.eslintrc.cjs b/.eslintrc.cjs
@@ -24,8 +24,16 @@ module.exports = {
         allow: [],
         depConstraints: [
           {
-            sourceTag: '*',
-            onlyDependOnLibsWithTags: ['*'],
+            sourceTag: 'scope:shared',
+            onlyDependOnLibsWithTags: ['scope:shared'],
+          },
+          {
+            sourceTag: 'scope:backend',
+            onlyDependOnLibsWithTags: ['scope:shared', 'scope:backend'],
+          },
+          {
+            sourceTag: 'scope:frontend',
+            onlyDependOnLibsWithTags: ['scope:shared', 'scope:frontend'],
           },
         ],
       },

diff --git a/.github/workflows/actions/nx-affected/action.yaml b/.github/workflows/actions/nx-affected/action.yaml
@@ -0,0 +1,21 @@
+name: Nx Affected CI
+inputs:
+  parallel:
+    required: false
+    types: [number]
+    default: 3
+  tag:
+    required: false
+    types: [string]
+  tasks:
+    required: true
+    types: [string]
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get last successful commit
+      uses: nrwl/nx-set-shas@v4
+    - name: Run affected command
+      shell: bash
+      run: npx nx affected --nxBail --configuration=ci -t=${{ inputs.tasks }} --parallel=${{ inputs.parallel }} --exclude='*,!tag:${{ inputs.tag }}'
diff --git a/.github/workflows/actions/task-cache/action.yaml b/.github/workflows/actions/task-cache/action.yaml
@@ -0,0 +1,29 @@
+name: Restore Tasks Cache CI
+inputs:
+  tag:
+    required: false
+    types: [string]
+  tasks:
+    required: false
+    types: [string]
+    default: all
+  suffix:
+    required: false
+    types: [string]
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compute tasks key
+      id: tasks-key
+      shell: bash
+      run: echo "key=${{ inputs.tasks }}" | tr , - >> $GITHUB_OUTPUT
+    - name: Restore tasks cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          .cache
+          .nx/cache
+        key: tasks-cache-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-${{ github.sha }}
+        restore-keys: |
+          tasks-cache-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-
diff --git a/.github/workflows/actions/yarn-install/action.yaml b/.github/workflows/actions/yarn-install/action.yaml
@@ -0,0 +1,23 @@
+name: Yarn Install
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Node.js and get yarn cache
+      uses: actions/setup-node@v3
+      with:
+        node-version: "18"
+        cache: yarn
+    - name: Cache node modules
+      id: node-modules-cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          node_modules
+          packages/*/node_modules
+        key: root-node_modules-${{ hashFiles('yarn.lock') }}
+        restore-keys: root-node_modules-
+    - name: Install Dependencies
+      shell: bash
+      run: yarn --immutable --check-cache
+      if: steps.node-modules-cache.outputs.cache-hit != 'true'
diff --git a/.github/workflows/ci-chromatic.yaml b/.github/workflows/ci-chromatic.yaml
@@ -27,17 +27,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Front / Write .env
         run: |
           cd packages/twenty-front
           touch .env
           echo "REACT_APP_SERVER_BASE_URL: $REACT_APP_SERVER_BASE_URL" >> .env
-      - name: Front / Install Dependencies
-        run: yarn
       - name: Publish to Chromatic
         run: |
           cd packages/twenty-front

diff --git a/.github/workflows/ci-chrome-extension.yaml b/.github/workflows/ci-chrome-extension.yaml
@@ -15,33 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  chrome-extension-yarn-install:
-    runs-on: ci-8-cores
-    env:
-      VITE_SERVER_BASE_URL: http://localhost:3000
-      VITE_FRONT_BASE_URL: http://localhost:3001
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache chrome extension node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-chrome-extension/node_modules
-          key: chrome-extension-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: chrome-extension-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Chrome Extension / Install Dependencies
-        run: yarn
   chrome-extension-build:
-    needs: chrome-extension-yarn-install
     runs-on: ubuntu-latest
     env:
       VITE_SERVER_BASE_URL: http://localhost:3000
@@ -52,21 +26,7 @@ jobs:
         with:
             access_token: ${{ github.token }}
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache chrome extension node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-chrome-extension/node_modules
-          key: chrome-extension-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: chrome-extension-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Chrome Extension / Run build
         run: npx nx build twenty-chrome-extension
diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
@@ -21,12 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Docs / Install Dependencies
-        run: yarn
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Docs / Build Documentation
         run: npx nx build twenty-docs
   vale:

diff --git a/.github/workflows/ci-front.yaml b/.github/workflows/ci-front.yaml
@@ -17,167 +17,63 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  front-yarn-install:
-    runs-on: ci-8-cores
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Front / Install Dependencies
-        run: yarn
-  front-pages-sb-test:
-    needs: front-yarn-install
+  front-sb-test:
     runs-on: ci-8-cores
+    strategy:
+      matrix:
+        storybook_scope: [pages, modules]
     env:
       REACT_APP_SERVER_BASE_URL: http://localhost:3000
+      STORYBOOK_SCOPE: ${{ matrix.storybook_scope }}
     steps:
       - name: Cancel Previous Runs
         uses: styfle/[email protected]
         with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Front / Write .env
-        run: |
-          cd packages/twenty-front
-          cp .env.example .env
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
+          access_token: ${{ github.token }}
+      - name: Fetch local actions
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Install Playwright
         run: cd packages/twenty-front && npx playwright install
-      - name: Run storybook tests
-        run: |
-          npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-            "STORYBOOK_SCOPE=pages npx nx run twenty-front:storybook:static:ci" \
-            "npx wait-on tcp:6006 && STORYBOOK_SCOPE=pages npx nx run twenty-front:storybook:test"
-  front-modules-sb-test:
-    needs: front-yarn-install
-    runs-on: ci-4-cores
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
+      - name: Front / Restore Storybook Task Cache
+        uses: ./.github/workflows/actions/task-cache
         with:
-          node-version: "18"
+          tag: scope:frontend
+          tasks: storybook:build
+          suffix: _${{ matrix.storybook_scope }}
       - name: Front / Write .env
-        run: |
-          cd packages/twenty-front
-          cp .env.example .env
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Install Playwright
-        run: cd packages/twenty-front && npx playwright install
+        run: npx nx reset:env twenty-front
       - name: Run storybook tests
         run: |
           npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-            "STORYBOOK_SCOPE=modules npx nx run twenty-front:storybook:static:ci" \
-            "npx wait-on tcp:6006 && STORYBOOK_SCOPE=modules npx nx run twenty-front:storybook:test"
-  front-lint-tsc:
-    needs: front-yarn-install
-    runs-on: ubuntu-latest
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: UI / Run linter
-        run: npx nx lint twenty-ui
-      - name: UI / Run Typescript Check
-        run: npx nx run twenty-ui:typecheck:ci
-      - name: Front / Run linter
-        run: npx nx run twenty-front:lint:ci
-      - name: Front / Run Typescript Check
-        run: npx nx run twenty-front:typecheck:ci
-  front-jest:
-    needs: front-yarn-install
+            "npx nx storybook:static twenty-front" \
+            "npx wait-on tcp:6006 && npx nx storybook:test twenty-front"
+  front-task:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        task: [lint, typecheck, test]
     env:
       REACT_APP_SERVER_BASE_URL: http://localhost:3000
     steps:
       - name: Cancel Previous Runs
         uses: styfle/[email protected]
         with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
+          access_token: ${{ github.token }}
+      - name: Fetch local actions and base branch history
+        uses: actions/checkout@v4
         with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
+          fetch-depth: 0
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
+      - name: Front / Restore Task Cache
+        uses: ./.github/workflows/actions/task-cache
         with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
+          tag: scope:frontend
+          tasks: ${{ matrix.task }}
+      - name: Front / Run task
+        uses: ./.github/workflows/actions/nx-affected
         with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Front / Run jest
-        run: npx nx test twenty-front --configuration=ci
+          tag: scope:frontend
+          tasks: ${{ matrix.task }}