[Security Solution][Endpoint] Combine the Endpoint management cypress test suites into a single configuration/run

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -1,18 +1,6 @@
 steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows.sh
     label: 'Defend Workflows Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/defend_workflows_vagrant.sh
-    label: 'Defend Workflows Endpoint Cypress Tests'
     agents:
       queue: n2-4-virt
     depends_on: build
@@ -35,15 +23,4 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/defend_workflows_vagrant_serverless.sh
-    label: 'Defend Workflows Endpoint Cypress Tests on Serverless'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 120
-    parallelism: 5
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
 

x-pack/plugins/security_solution/package.json

@@ -13,15 +13,12 @@
     "cypress:dw": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress.config.ts --ftr-config-file ../../test/defend_workflows_cypress/cli_config",
     "cypress:dw:open": "yarn cypress:dw open",
     "cypress:dw:run": "yarn cypress:dw run",
-    "cypress:dw:endpoint": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_endpoint.config.ts --ftr-config-file ../../test/defend_workflows_cypress/endpoint_config",
-    "cypress:dw:endpoint:run": "yarn cypress:dw:endpoint run",
-    "cypress:dw:endpoint:open": "yarn cypress:dw:endpoint open",
     "cypress:dw:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_serverless.config.ts --ftr-config-file ../../../x-pack/test_serverless/functional/test_suites/security/cypress/security_config",
     "cypress:dw:serverless:open": "yarn cypress:dw:serverless open",
     "cypress:dw:serverless:run": "yarn cypress:dw:serverless run",
-    "cypress:dw:endpoint:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_endpoint_serverless.config.ts --ftr-config-file ../../test/defend_workflows_cypress/endpoint_serverless_config",
-    "cypress:dw:endpoint:serverless:open": "yarn cypress:dw:endpoint:serverless open",
-    "cypress:dw:endpoint:serverless:run": "yarn cypress:dw:endpoint:serverless run",
+    "cypress:dw:endpoint": "echo '\n** WARNING **: Run script `cypress:dw:endpoint` no longer valid! Use `cypress:dw` instead\n'",
+    "cypress:dw:endpoint:run": "echo '\n** WARNING **: Run script `cypress:dw:endpoint:run` no longer valid! Use `cypress:dw:run` instead\n'",
+    "cypress:dw:endpoint:open": "echo '\n** WARNING **: Run script `cypress:dw:endpoint:open` no longer valid! Use `cypress:dw:open` instead\n'",
     "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && yarn junit:transform && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
     "test:generate": "node scripts/endpoint/resolver_generator",
     "mappings:generate": "node scripts/mappings/mappings_generator",

x-pack/plugins/security_solution/public/management/cypress/README.md

@@ -134,67 +134,12 @@ yarn --cwd x-pack/plugins/security_solution cypress:dw:open
 yarn --cwd x-pack/plugins/security_solution cypress:dw:serverless:open
 ```
 
-For developing/debugging tests against real endpoint please use:
-
-Endpoint tests require [Multipass](https://multipass.run/) to be installed on your machine.
-
-```shell
-# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
-yarn kbn bootstrap && node scripts/build_kibana_platform_plugins
-
-# launch the cypress test runner against ESS
-yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:open
-
-# or against Serverless
-yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:serverless:open
-```
-
-#### Cypress (interactive) with real Endpoints using Vagrant
-
-```shell
-# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
-yarn kbn bootstrap && node scripts/build_kibana_platform_plugins
-
-# launch the cypress test runner with real endpoint
-cd x-pack/plugins/security_solution
-export CI=true
-yarn cypress:dw:endpoint:open
-````
-
-Note that you can select the browser you want to use on the top right side of the interactive runner.
-
-#### Cypress against REAL Endpoint + Headless (Chrome)
-
-This requires some additional setup as mentioned in the [pre-requisites](#pre-requisites) section.
-
-Endpoint tests require [Multipass](https://multipass.run/) to be installed on your machine.
-
-```shell
-# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
-yarn kbn bootstrap && node scripts/build_kibana_platform_plugins
-
-# launch the cypress test runner with real endpoint against ESS
-yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:run
-
-# or against Serverless
-yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:serverless:run
-```
-
 ## Folder Structure
 
 ### e2e/
 
 Contains all the tests. Within it are two sub-folders:
 
-#### cypress/endpoint
-
-Contains all the tests that are executed against real endpoints.
-
-#### cypress/mocked_data
-
-Contains all the tests that are executed against mocked endpoint and run on CI. If you want to add tests that run on CI
-then this is where you should add those.
-
 ### integration/
 
 Cypress convention. Contains the specs that are going to be executed.
@@ -212,7 +157,6 @@ Directory also holds Cypress Plugins that are then initialized via `setupNodeEve
 ### screens/
 
 Contains the elements we want to interact with within our tests.
-
 Each file inside the screens folder represents a screen in our application.
 
 ### tasks/

x-pack/plugins/security_solution/public/management/cypress/cypress.config.ts

@@ -8,26 +8,10 @@
 import { defineCypressConfig } from '@kbn/cypress-config';
 import { getCypressBaseConfig } from './cypress_base.config';
 
-import { dataLoaders } from './support/data_loaders';
-
-export default defineCypressConfig({
-  ...getCypressBaseConfig(),
-
-  env: {
-    ...getCypressBaseConfig().env,
-
-    grepTags: '@ess',
-  },
-
-  e2e: {
-    ...getCypressBaseConfig().e2e,
-
-    specPattern: 'public/management/cypress/e2e/mocked_data/',
-    setupNodeEvents: (on, config) => {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      require('@cypress/grep/src/plugin')(config);
-
-      return dataLoaders(on, config);
+export default defineCypressConfig(
+  getCypressBaseConfig({
+    env: {
+      grepTags: '@ess',
     },
-  },
-});
+  })
+);

x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts

@@ -5,50 +5,82 @@
  * 2.0.
  */
 
-export const getCypressBaseConfig = () => ({
-  reporter: '../../../../node_modules/cypress-multi-reporters',
-  reporterOptions: {
-    configFile: './public/management/reporter_config.json',
-  },
-
-  defaultCommandTimeout: 60000,
-  execTimeout: 120000,
-  pageLoadTimeout: 12000,
-
-  retries: {
-    runMode: 1,
-    openMode: 0,
-  },
-
-  screenshotsFolder:
-    '../../../target/kibana-security-solution/public/management/cypress/screenshots',
-  trashAssetsBeforeRuns: false,
-  video: false,
-  viewportHeight: 900,
-  viewportWidth: 1440,
-  experimentalStudio: true,
-
-  env: {
-    KIBANA_URL: 'http://localhost:5601',
-    ELASTICSEARCH_URL: 'http://localhost:9200',
-    FLEET_SERVER_URL: 'https://localhost:8220',
-
-    // Username/password used for both elastic and kibana
-    KIBANA_USERNAME: 'system_indices_superuser',
-    KIBANA_PASSWORD: 'changeme',
-    ELASTICSEARCH_USERNAME: 'system_indices_superuser',
-    ELASTICSEARCH_PASSWORD: 'changeme',
-
-    // grep related configs
-    grepFilterSpecs: true,
-    grepOmitFiltered: true,
-  },
-
-  e2e: {
-    // baseUrl: To override, set Env. variable `CYPRESS_BASE_URL`
-    baseUrl: 'http://localhost:5601',
-
-    supportFile: 'public/management/cypress/support/e2e.ts',
-    experimentalRunAllSpecs: true,
-  },
-});
+import { merge } from 'lodash';
+import { dataLoaders, dataLoadersForRealEndpoints } from './support/data_loaders';
+import { responseActionTasks } from './support/response_actions';
+
+export const getCypressBaseConfig = (
+  overrides: Cypress.ConfigOptions = {}
+): Cypress.ConfigOptions => {
+  return merge(
+    {
+      reporter: '../../../../node_modules/cypress-multi-reporters',
+      reporterOptions: {
+        configFile: './public/management/reporter_config.json',
+      },
+
+      defaultCommandTimeout: 60000,
+      execTimeout: 120000,
+      pageLoadTimeout: 12000,
+
+      retries: {
+        runMode: 1,
+        openMode: 0,
+      },
+
+      screenshotsFolder:
+        '../../../target/kibana-security-solution/public/management/cypress/screenshots',
+      trashAssetsBeforeRuns: false,
+      video: false,
+      viewportHeight: 900,
+      viewportWidth: 1440,
+      experimentalStudio: true,
+
+      env: {
+        'cypress-react-selector': {
+          root: '#security-solution-app',
+        },
+        KIBANA_URL: 'http://localhost:5601',
+        ELASTICSEARCH_URL: 'http://localhost:9200',
+        FLEET_SERVER_URL: 'https://localhost:8220',
+        KIBANA_USERNAME: 'system_indices_superuser',
+        KIBANA_PASSWORD: 'changeme',
+        ELASTICSEARCH_USERNAME: 'system_indices_superuser',
+        ELASTICSEARCH_PASSWORD: 'changeme',
+
+        // grep related configs
+        grepFilterSpecs: true,
+        grepOmitFiltered: true,
+      },
+
+      e2e: {
+        // baseUrl: To override, set Env. variable `CYPRESS_BASE_URL`
+        baseUrl: 'http://localhost:5601',
+        supportFile: 'public/management/cypress/support/e2e.ts',
+        specPattern: 'public/management/cypress/e2e/**/*.cy.{js,jsx,ts,tsx}',
+        experimentalRunAllSpecs: true,
+        setupNodeEvents: (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => {
+          dataLoaders(on, config);
+
+          // skip dataLoadersForRealEndpoints() if running in serverless
+          // https://github.com/elastic/security-team/issues/7467
+          // Once we are able to run Fleet server in serverless mode (see: https://github.com/elastic/kibana/pull/166183)
+          // this `if()` statement needs to be removed and `dataLoadersForRealEndpoints()` should
+          // just be called without having any checks around it.
+          if (!config.env.IS_SERVERLESS) {
+            // Data loaders specific to "real" Endpoint testing
+            dataLoadersForRealEndpoints(on, config);
+          }
+
+          responseActionTasks(on, config);
+
+          // eslint-disable-next-line @typescript-eslint/no-var-requires
+          require('@cypress/grep/src/plugin')(config);
+
+          return config;
+        },
+      },
+    },
+    overrides
+  );
+};