ci: implement aws tests on prs (#2406)

* test(cli): fix existing lambda test

* ci(docker): allow publishing of ecs worker image from prs

* ci(test): add workflow for running aws tests when pr is labeled

* ci: pin check-user-permission action to specific sha

.github/workflows/docker-ecs-worker-image.yml

@@ -5,6 +5,15 @@ on:
   push:
     branches:
       - main
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Branch ref to checkout. Needed for pull_request_target to be able to pull correct ref.'
+        type: string
+    secrets:
+      ECR_WORKER_IMAGE_PUSH_ROLE_ARN:
+        description: 'ARN of the IAM role to assume to push the image to ECR.'
+        required: true
 
 permissions:
   id-token: write
@@ -16,6 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          ref: ${{ inputs.ref || null }}
           fetch-depth: 0
 
       - name: Show git ref

.github/workflows/run-aws-tests.yml

@@ -0,0 +1,71 @@
+name: Run AWS tests
+
+on:
+  pull_request_target:
+    branches: [main]
+    #opened, reopened and synchronize will cause the workflow to fail on forks due to permissions
+    #once labeled, that will then be overridden by the is-collaborator job
+    types: [opened, labeled, synchronize, reopened]
+
+jobs:
+  is-collaborator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get User Permission
+        id: checkAccess
+        uses: actions-cool/check-user-permission@cd622002ff25c2311d2e7fb82107c0d24be83f9b
+        with:
+          require: write
+          username: ${{ github.actor }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check User Permission
+        if: steps.checkAccess.outputs.require-result == 'false'
+        run: |
+          echo "${{ github.actor }} does not have permissions on this repo."
+          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
+          exit 1
+
+  publish-branch-image:
+    if: contains( github.event.pull_request.labels.*.name, 'run-aws-tests' )
+    needs: is-collaborator
+    uses: ./.github/workflows/docker-ecs-worker-image.yml
+    permissions:
+      contents: read
+      id-token: write
+    secrets:
+      ECR_WORKER_IMAGE_PUSH_ROLE_ARN: ${{ secrets.ECR_WORKER_IMAGE_PUSH_ROLE_ARN }}
+    with:
+      ref: ${{ github.event.pull_request.head.sha || null }} # this should only be run with this ref if is-collaborator has been run and passed
+
+  run-tests:
+    if: contains( github.event.pull_request.labels.*.name, 'run-aws-tests' )
+    needs: publish-branch-image
+    timeout-minutes: 40
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha || null }} # this should only be run with this ref if is-collaborator has been run and passed
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        env:
+          SHOW_STACK_TRACE: true
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.ARTILLERY_AWS_CLI_ROLE_ARN_TEST1 }}
+          role-session-name: OIDCSession
+          mask-aws-account-id: true
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18.x
+      - run: npm install
+      - run: npm run build
+      - run: npm run test:aws --workspace artillery
+        env:
+          FORCE_COLOR: 1
+          ECR_IMAGE_VERSION: ${{ github.sha }} # the image is published with the sha of the commit within this repo

packages/artillery/package.json

@@ -46,7 +46,7 @@
     "test:unit": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 --color test/unit/*.test.js",
     "test:acceptance": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --no-coverage --color test/testcases/plugins/*.test.js",
     "test": "npm run test:unit && npm run test:acceptance",
-    "test:cloud": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=300 test/cloud-e2e/*.test.js",
+    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=600 test/cloud-e2e/**/*.test.js",
     "lint": "eslint --ext \".js,.ts,.tsx\" .",
     "lint-fix": "npm run lint -- --fix"
   },

packages/artillery/test/scripts/blitz/config.yml → packages/artillery/test/cloud-e2e/lambda/fixtures/quick-loop-with-csv/config.yml

@@ -1,13 +1,11 @@
 config:
   target: http://asciiart.artillery.io:8080
-  processor: ./helpers-blitz.js
+  processor: ./helpers.js
   payload:
     - path: ./test.csv
       fields:
         - number
         - name
-  plugins:
-    metrics-by-endpoint: {}
   phases:
     - duration: 30
       arrivalRate: 1

packages/artillery/test/scripts/blitz/test.csv → packages/artillery/test/cloud-e2e/lambda/fixtures/quick-loop-with-csv/test.csv

@@ -1,3 +1,3 @@
 pony,tiki
 dog,mali
-cat,faye
+cat,faye

packages/artillery/test/cloud-e2e/lambda/run-lambda.test.js

@@ -0,0 +1,26 @@
+const tap = require('tap');
+const { execute } = require('../../cli/_helpers.js');
+
+tap.test('Run a test on AWS Lambda', async (t) => {
+  const [exitCode, output] = await execute([
+    'run:lambda',
+    '--count',
+    '10',
+    '--region',
+    'eu-west-1',
+    '--config',
+    './test/cloud-e2e/lambda/fixtures/quick-loop-with-csv/config.yml',
+    './test/cloud-e2e/lambda/fixtures/quick-loop-with-csv/blitz.yml'
+  ]);
+
+  t.equal(exitCode, 0, 'CLI should exit with code 0');
+
+  t.ok(
+    output.stdout.indexOf('Summary report') > 0,
+    'Should print summary report'
+  );
+  t.ok(
+    output.stdout.indexOf('http.codes.200') > 0,
+    'Should print http.codes.200'
+  );
+});