CI: Run e2e tests in parallel using multiple suites (grafana#41748)

* Split suite1 to multiple e2e test suites

* Update drone yaml

* Add missing arg

* Extract cypress install step

* Change image on e2e tests step

* Sync with main

* Add cypress cache volume

* Try with grafana/ci-e2e:12.19.0-1 image

* Update grabpl version to 2.7.2

* Revert a11y failure back to 'always'

e2e/run-suite

@@ -8,11 +8,12 @@ PORT=${PORT:-$DEFAULT_PORT}
 
 echo -e "Starting Cypress scenarios"
 
+args=("$@")
+
 CMD="start"
 PARAMS=""
 SLOWMO=0
 URL=${BASE_URL:-"http://$HOST:$PORT"}
-SUITE=${SUITE:-$DEFAULT_SUITE}
 
 if [ "$1" == "debug" ]; then
     echo -e "Debug mode"
@@ -28,5 +29,5 @@ fi
 cd packages/grafana-e2e
 
 yarn $CMD --env BASE_URL=$URL,SLOWMO=$SLOWMO \
-  --config defaultCommandTimeout=30000,integrationFolder=../../e2e/$SUITE/specs,screenshotsFolder=../../e2e/$SUITE/screenshots,videosFolder=../../e2e/$SUITE/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false,videoUploadOnPasses=false \
+  --config defaultCommandTimeout=30000,integrationFolder=../../e2e/"${args[0]}"/specs,screenshotsFolder=../../e2e"${args[0]}"/screenshots,videosFolder=../../e2e/"${args[0]}"/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false,videoUploadOnPasses=false \
   $PARAMS

e2e/variables

@@ -8,4 +8,3 @@ DEFAULT_PACKAGE_FILE=dist/grafana-*linux-amd64.tar.gz
 PROV_DIR=$RUNDIR/conf/provisioning
 DEFAULT_HOST=localhost
 DEFAULT_PORT=3001
-DEFAULT_SUITE=suite1

scripts/drone/pipelines/main.star

@@ -14,6 +14,7 @@ load(
     'build_frontend_step',
     'build_plugins_step',
     'package_step',
+    'install_cypress_step',
     'e2e_tests_server_step',
     'e2e_tests_step',
     'build_storybook_step',
@@ -86,8 +87,12 @@ def get_steps(edition, is_downstream=False):
     # Insert remaining steps
     steps.extend([
         package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
+        install_cypress_step(),
         e2e_tests_server_step(edition=edition),
-        e2e_tests_step(edition=edition),
+        e2e_tests_step('dashboards-suite', edition=edition),
+        e2e_tests_step('smoke-tests-suite', edition=edition),
+        e2e_tests_step('panels-suite', edition=edition),
+        e2e_tests_step('various-suite', edition=edition),
         build_storybook_step(edition=edition, ver_mode=ver_mode),
         publish_storybook_step(edition=edition, ver_mode=ver_mode),
         test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
@@ -113,6 +118,10 @@ def get_steps(edition, is_downstream=False):
             package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64'], is_downstream=is_downstream),
             e2e_tests_server_step(edition=edition2, port=3002),
             e2e_tests_step(edition=edition2, port=3002),
+            e2e_tests_step('dashboards-suite', edition=edition2, port=3002),
+            e2e_tests_step('smoke-tests-suite', edition=edition2, port=3002),
+            e2e_tests_step('panels-suite', edition=edition2, port=3002),
+            e2e_tests_step('various-suite', edition=edition2, port=3002),
             upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
             upload_cdn_step(edition=edition2)
         ])

scripts/drone/pipelines/pr.star

@@ -14,6 +14,7 @@ load(
     'test_backend_integration_step',
     'test_frontend_step',
     'package_step',
+    'install_cypress_step',
     'e2e_tests_server_step',
     'e2e_tests_step',
     'build_storybook_step',
@@ -86,8 +87,12 @@ def pr_pipelines(edition):
     # Insert remaining build_steps
     build_steps.extend([
         package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=variants),
+        install_cypress_step(),
         e2e_tests_server_step(edition=edition),
-        e2e_tests_step(edition=edition),
+        e2e_tests_step('dashboards-suite', edition=edition),
+        e2e_tests_step('smoke-tests-suite', edition=edition),
+        e2e_tests_step('panels-suite', edition=edition),
+        e2e_tests_step('various-suite', edition=edition),
         build_storybook_step(edition=edition, ver_mode=ver_mode),
         test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
         build_frontend_docs_step(edition=edition),
@@ -105,6 +110,10 @@ def pr_pipelines(edition):
             package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64']),
             e2e_tests_server_step(edition=edition2, port=3002),
             e2e_tests_step(edition=edition2, port=3002),
+            e2e_tests_step('dashboards-suite', edition=edition2, port=3002),
+            e2e_tests_step('smoke-tests-suite', edition=edition2, port=3002),
+            e2e_tests_step('panels-suite', edition=edition2, port=3002),
+            e2e_tests_step('various-suite', edition=edition2, port=3002),
         ])
 
     trigger = {

scripts/drone/pipelines/release.star

@@ -17,6 +17,7 @@ load(
     'build_frontend_step',
     'build_plugins_step',
     'package_step',
+    'install_cypress_step',
     'e2e_tests_server_step',
     'e2e_tests_step',
     'build_storybook_step',
@@ -114,8 +115,12 @@ def get_steps(edition, ver_mode):
     # Insert remaining steps
     build_steps.extend([
         package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2),
+        install_cypress_step(),
         e2e_tests_server_step(edition=edition),
-        e2e_tests_step(edition=edition, tries=3),
+        e2e_tests_step('dashboards-suite', edition=edition, tries=3),
+        e2e_tests_step('smoke-tests-suite', edition=edition, tries=3),
+        e2e_tests_step('panels-suite', edition=edition, tries=3),
+        e2e_tests_step('various-suite', edition=edition, tries=3),
         copy_packages_for_docker_step(),
         build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
         build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
@@ -145,7 +150,10 @@ def get_steps(edition, ver_mode):
         publish_steps.extend([
             package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64']),
             e2e_tests_server_step(edition=edition2, port=3002),
-            e2e_tests_step(edition=edition2, port=3002, tries=3),
+            e2e_tests_step('dashboards-suite', edition=edition2, port=3002, tries=3),
+            e2e_tests_step('smoke-tests-suite', edition=edition2, port=3002, tries=3),
+            e2e_tests_step('panels-suite', edition=edition2, port=3002, tries=3),
+            e2e_tests_step('various-suite', edition=edition2, port=3002, tries=3),
             upload_cdn_step(edition=edition2),
         ])
         if should_upload:

scripts/drone/steps/lib.star

@@ -1,6 +1,6 @@
 load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
 
-grabpl_version = '2.7.1'
+grabpl_version = '2.7.2'
 build_image = 'grafana/build-container:1.4.6'
 publish_image = 'grafana/grafana-ci-deploy:1.3.1'
 grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
@@ -276,7 +276,10 @@ def publish_storybook_step(edition, ver_mode):
         'image': publish_image,
         'depends_on': [
             'build-storybook',
-            'end-to-end-tests',
+            'end-to-end-tests-dashboards-suite',
+            'end-to-end-tests-panels-suite',
+            'end-to-end-tests-smoke-tests-suite',
+            'end-to-end-tests-various-suite',
         ],
         'environment': {
             'GCP_KEY': from_secret('gcp_key'),
@@ -650,24 +653,40 @@ def e2e_tests_server_step(edition, port=3001):
         ],
     }
 
+def install_cypress_step():
+    return {
+        'name': 'cypress',
+        'image': 'grafana/ci-e2e:12.19.0-1',
+        'depends_on': [
+            'package',
+            ],
+        'commands': [
+            'yarn run cypress install',
+        ],
+        'volumes': [{
+            'name': 'cypress_cache',
+            'path': '/root/.cache/Cypress'
+        }],
+    }
 
-def e2e_tests_step(edition, port=3001, tries=None):
-    cmd = './bin/grabpl e2e-tests --port {}'.format(port)
+def e2e_tests_step(suite, edition, port=3001, tries=None):
+    cmd = './bin/grabpl e2e-tests --port {} --suite {}'.format(port, suite)
     if tries:
         cmd += ' --tries {}'.format(tries)
     return {
-        'name': 'end-to-end-tests' + enterprise2_suffix(edition),
+        'name': 'end-to-end-tests-{}'.format(suite) + enterprise2_suffix(edition),
         'image': 'grafana/ci-e2e:12.19.0-1',
         'depends_on': [
-            'end-to-end-tests-server' + enterprise2_suffix(edition),
+            'cypress',
         ],
         'environment': {
             'HOST': 'end-to-end-tests-server' + enterprise2_suffix(edition),
         },
+        'volumes': [{
+            'name': 'cypress_cache',
+            'path': '/root/.cache/Cypress'
+        }],
         'commands': [
-            # Have to re-install Cypress since it insists on searching for its binary beneath /root/.cache,
-            # even though the Yarn cache directory is beneath /usr/local/share somewhere
-            'yarn run cypress install',
             cmd,
         ],
     }
@@ -832,7 +851,10 @@ def release_canary_npm_packages_step(edition):
         'name': 'release-canary-npm-packages',
         'image': build_image,
         'depends_on': [
-            'end-to-end-tests',
+            'end-to-end-tests-dashboards-suite',
+            'end-to-end-tests-panels-suite',
+            'end-to-end-tests-smoke-tests-suite',
+            'end-to-end-tests-various-suite',
         ],
         'environment': {
             'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'),
@@ -862,7 +884,10 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
         cmd = './bin/grabpl upload-packages --edition {}{}'.format(edition, packages_bucket)
 
     dependencies = [
-        'end-to-end-tests' + enterprise2_suffix(edition),
+        'end-to-end-tests-dashboards-suite' + enterprise2_suffix(edition),
+        'end-to-end-tests-panels-suite' + enterprise2_suffix(edition),
+        'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition),
+        'end-to-end-tests-various-suite' + enterprise2_suffix(edition),
     ]
 
     if edition in ('enterprise', 'enterprise2'):

scripts/drone/utils/utils.star

@@ -37,6 +37,10 @@ def pipeline(
         'trigger': trigger,
         'services': services,
         'steps': steps,
+        'volumes': [{
+            'name': 'cypress_cache',
+            'temp': {},
+        }],
         'depends_on': depends_on,
     }
     pipeline.update(platform_conf)