ok-to-test-command #21319
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Appsmith External Integration Test Workflow | |
| on: | |
| # This workflow is only triggered by the ok to test command dispatch | |
| repository_dispatch: | |
| types: [ok-to-test-command] | |
| jobs: | |
| tag-check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tags: ${{ steps.checkAll.outputs.tags }} | |
| matrix: ${{ steps.checkAll.outputs.matrix }} | |
| steps: | |
| - name: Check if tags arg is present in the command | |
| id: checkTags | |
| run: | | |
| checkArg=`echo '${{toJSON(github.event.client_payload.slash_command.args.named)}}' | jq 'has("tags")'` | |
| echo "tags=$checkArg" >> $GITHUB_OUTPUT | |
| - name: Add a comment on the PR with tags documentation link | |
| if: steps.checkTags.outputs.tags != 'true' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| The provided command lacks any tags. Please execute '/ok-to-test' again, specifying the tags you want to include or use `/ok-to-test tags="@tag.All"` to run all specs. | |
| Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) | |
| - name: Stop the workflow run if tags are not present | |
| if: steps.checkTags.outputs.tags != 'true' | |
| run: exit 1 | |
| - name: Check if @tag.All is present in tags | |
| if: steps.checkTags.outputs.tags == 'true' | |
| id: checkAll | |
| run: | | |
| tags="${{ github.event.client_payload.slash_command.args.named.tags }}" | |
| if [[ ($tags == *"ALL"* || $tags == *"All"* || $tags == *"all"*) && $tags != *"@tag.All"* ]]; then | |
| echo "invalid_tags_all=$tags" >> $GITHUB_OUTPUT | |
| elif [[ $tags == *"@tag.All"* ]]; then | |
| echo "tags=" >> $GITHUB_OUTPUT | |
| echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT | |
| else | |
| echo "tags=$tags" >> $GITHUB_OUTPUT | |
| echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Add comment to use correct @tag.All format | |
| if: steps.checkAll.outputs.invalid_tags_all != '' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Please use `/ok-to-test tags="@tag.All"` to run all specs. | |
| Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) | |
| - name: Stop the workflow run if given @tag.All format is wrong | |
| if: steps.checkAll.outputs.invalid_tags_all != '' | |
| run: exit 1 | |
| # tags is set to empty string if @tag.All is given in step - Check if @tag.All is present in tags | |
| - name: Add suggestion in the PR on using @tag.All | |
| if: steps.checkAll.outputs.tags == '' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Whoa, @tag.All spotted in your test suite! 🚀 | |
| While @tag.All is cool, like a catch-all net, why not try specific tags? 🏷️ | |
| Narrow down your suite with specific tags for quicker and more accurate tests! 🚀 Less waiting, more zipping through tests like a ninja! | |
| Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) | |
| # This step creates a comment on the PR with a link to this workflow run. | |
| - name: Add a comment on the PR with link to workflow run | |
| if: steps.checkTags.outputs.tags == 'true' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Workflow: `${{ github.workflow }}`. | |
| Tags: `${{ steps.checkAll.outputs.tags }}`. | |
| server-build: | |
| needs: [tag-check] | |
| if: success() | |
| name: server-build | |
| uses: ./.github/workflows/server-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| skip-tests: true | |
| client-build: | |
| needs: [tag-check] | |
| if: success() | |
| name: client-build | |
| uses: ./.github/workflows/client-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| check-test-files: "true" | |
| rts-build: | |
| needs: [tag-check] | |
| if: success() | |
| name: rts-build | |
| uses: ./.github/workflows/rts-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| test-appsmithctl: | |
| needs: [tag-check] | |
| if: success() | |
| name: appsmithctl | |
| uses: ./.github/workflows/appsmithctl.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| build-docker-image: | |
| needs: [tag-check, client-build, server-build, rts-build] | |
| # Only run if the build step is successful | |
| if: success() | |
| name: build-docker-image | |
| uses: ./.github/workflows/build-docker-image.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| ci-test: | |
| needs: [tag-check, build-docker-image] | |
| # Only run if the build step is successful | |
| if: success() | |
| name: ci-test | |
| uses: ./.github/workflows/ci-test-custom-script.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| tags: ${{ needs.tag-check.outputs.tags }} | |
| matrix: ${{ needs.tag-check.outputs.matrix }} | |
| ci-test-result: | |
| needs: [ci-test] | |
| # Only run if the ci-test with matrices step is successful | |
| if: always() | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Dump the client payload context | |
| env: | |
| PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} | |
| run: echo "$PAYLOAD_CONTEXT" | |
| - name: Setup node | |
| if: needs.ci-test.result != 'success' | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: install pg | |
| if: needs.ci-test.result != 'success' | |
| run: npm install pg | |
| - name: Fetch the failed specs | |
| if: needs.ci-test.result != 'success' | |
| id: failed_specs | |
| env: | |
| DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} | |
| DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} | |
| DB_USER: ${{ secrets.CYPRESS_DB_USER }} | |
| DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} | |
| RUN_ID: ${{ github.run_id }} | |
| ATTEMPT_NUMBER: ${{ github.run_attempt }} | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const { Pool } = require("pg"); | |
| const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env | |
| const client = await new Pool({ | |
| user: DB_USER, | |
| host: DB_HOST, | |
| database: DB_NAME, | |
| password: DB_PWD, | |
| port: 5432, | |
| connectionTimeoutMillis: 60000, | |
| }).connect(); | |
| const result = await client.query( | |
| `SELECT DISTINCT name FROM public."specs" | |
| WHERE "matrixId" IN | |
| (SELECT id FROM public."matrix" | |
| WHERE "attemptId" = ( | |
| SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 | |
| ) | |
| ) AND status = 'fail'`, | |
| [RUN_ID, ATTEMPT_NUMBER], | |
| ); | |
| client.release(); | |
| return result.rows.map((spec) => spec.name); | |
| # In case for any ci job failure, create combined failed spec | |
| - name: combine all specs for CI | |
| id: combine_ci | |
| if: needs.ci-test.result != 'success' | |
| run: | | |
| failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n') | |
| while read -r line; do | |
| echo "$line" >> ~/combined_failed_spec_ci | |
| done <<< "$failed_specs" | |
| if [[ -z $(grep '[^[:space:]]' ~/combined_failed_spec_ci) ]] ; then | |
| echo "specs_failed=0" >> $GITHUB_OUTPUT | |
| else | |
| echo "specs_failed=1" >> $GITHUB_OUTPUT | |
| fi | |
| # Upload combined failed CI spec list to a file | |
| # This is done for debugging. | |
| - name: upload combined failed spec | |
| if: needs.ci-test.result != 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: combined_failed_spec_ci | |
| path: ~/combined_failed_spec_ci | |
| - name: Get Latest flaky Tests | |
| shell: bash | |
| run: | | |
| curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures | |
| # Verify CI test failures against known failures | |
| - name: Verify CI test failures against known failures | |
| if: needs.ci-test.result != 'success' | |
| shell: bash | |
| run: | | |
| new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>" | |
| echo "$new_failed_spec_env" | |
| echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV | |
| echo "$new_failed_spec_env" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Add a comment on the PR with new CI failures | |
| if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank"> Click here!</a> | |
| The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} | |
| To know the list of identified flaky tests - <a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">Refer here</a> | |
| - name: Add a comment on the PR when ci-test is failed but no specs found | |
| if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a> | |
| It seems like **no tests ran** 😔. We are not able to recognize it, please check workflow <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">here.</a> | |
| - name: Add a comment on the PR when ci-test is success | |
| if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a> | |
| All cypress tests have passed 🎉🎉🎉 | |
| # Update check run called "ci-test-result" | |
| - name: Mark ci-test-result job as complete | |
| uses: actions/github-script@v6 | |
| id: update-check-run | |
| if: ${{ always() }} | |
| env: | |
| run_id: ${{ github.run_id }} | |
| repository: ${{ github.repository }} | |
| number: ${{ github.event.client_payload.pull_request.number }} | |
| job: ${{ github.job }} | |
| # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run | |
| conclusion: ${{ job.status }} | |
| matrix_result: ${{ toJson(needs.ci-test) }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { data: pull } = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: process.env.number | |
| }); | |
| const ref = pull.head.sha; | |
| const { data: checks } = await github.rest.checks.listForRef({ | |
| ...context.repo, | |
| ref | |
| }); | |
| const check = checks.check_runs.filter(c => c.name === process.env.job); | |
| if(check.length == 0) { | |
| try { | |
| console.log("Going to create the check run: ci-test-result"); | |
| const head_sha = pull.head.sha; | |
| const { data: completed_at } = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: head_sha, | |
| name: process.env.job, | |
| status: 'completed', | |
| conclusion: JSON.parse(process.env.matrix_result).result, | |
| output: { | |
| title: "Integration tests result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| console.log({ completed_at }); | |
| return completed_at; | |
| } catch(e) { | |
| console.error("Error while creating the check run: ci-test-result"); | |
| console.error({ error: e.message }); | |
| } | |
| } else { | |
| try { | |
| console.log("Going to update the check run ci-test-result with id: " + check[0].id); | |
| const { data: result } = await github.rest.checks.update({ | |
| ...context.repo, | |
| check_run_id: check[0].id, | |
| status: 'completed', | |
| conclusion: JSON.parse(process.env.matrix_result).result, | |
| output: { | |
| title: "Integration tests result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| console.log({ result }); | |
| return result; | |
| } catch(e) { | |
| console.error("Error while updating the check run: ci-test-result for check_id: " + check[0].id); | |
| console.error({ error: e.message }); | |
| } | |
| } | |
| - name: Dump the client payload context | |
| env: | |
| PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} | |
| run: echo "$PAYLOAD_CONTEXT" | |
| - name: Check ci-test set status | |
| if: needs.ci-test.result != 'success' | |
| run: exit 1 | |
| package: | |
| needs: [ci-test] | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: app/client | |
| # Run this job only if all the previous steps are a success and the reference if the release or master branch | |
| if: success() && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/master') | |
| steps: | |
| # Update check run called "package" | |
| - name: Mark package job as complete | |
| uses: actions/github-script@v6 | |
| id: update-check-run | |
| if: ${{ always() }} | |
| env: | |
| run_id: ${{ github.run_id }} | |
| repository: ${{ github.repository }} | |
| number: ${{ github.event.client_payload.pull_request.number }} | |
| job: ${{ github.job }} | |
| # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run | |
| conclusion: ${{ job.status }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { data: pull } = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: process.env.number | |
| }); | |
| const ref = pull.head.sha; | |
| const { data: checks } = await github.rest.checks.listForRef({ | |
| ...context.repo, | |
| ref | |
| }); | |
| const check = checks.check_runs.filter(c => c.name === process.env.job); | |
| if(check.length == 0) { | |
| const head_sha = pull.head.sha; | |
| const { data: completed_at } = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: head_sha, | |
| name: process.env.job, | |
| status: 'completed', | |
| conclusion: process.env.conclusion, | |
| output: { | |
| title: "Package result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| return completed_at; | |
| } else { | |
| const { data: result } = await github.rest.checks.update({ | |
| ...context.repo, | |
| check_run_id: check[0].id, | |
| status: 'completed', | |
| conclusion: process.env.conclusion, | |
| output: { | |
| title: "Package result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| return result; | |
| } |