[Security][Quality Gate] EDR Workflows Integration

.buildkite/pipelines/security_solution/security_solution_defend_workflows.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 300
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-defend-workflows-serverless-cypress
+
+cd x-pack/plugins/security_solution
+set +e
+
+QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+
+CLOUD_QA_API_KEY=$QA_API_KEY yarn cypress:dw:qa:serverless:run; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/pipeline.sh

@@ -2,4 +2,4 @@
 
 set -euo pipefail
 
-echo "Running the EDR-Workflows testing for Kibana"
+ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/pipeline.ts

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/pipeline.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+
+const getPipeline = (filename: string, removeSteps = true) => {
+  const str = fs.readFileSync(filename).toString();
+  return removeSteps ? str.replace(/^steps:/, '') : str;
+};
+
+const uploadPipeline = (pipelineContent: string | object) => {
+  const str =
+    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
+
+  execSync('buildkite-agent pipeline upload', {
+    input: str,
+    stdio: ['pipe', 'inherit', 'inherit'],
+  });
+};
+
+(async () => {
+  try {
+    const pipeline = [];
+
+    pipeline.push(
+      getPipeline(
+        '.buildkite/pipelines/security_solution/security_solution_defend_workflows.yml',
+        false
+      )
+    );
+    // remove duplicated steps
+    uploadPipeline([...new Set(pipeline)].join('\n'));
+  } catch (ex) {
+    console.error('PR pipeline generation error', ex.message);
+    process.exit(1);
+  }
+})();

x-pack/plugins/security_solution/package.json

@@ -16,6 +16,8 @@
     "cypress:dw:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_serverless.config.ts --ftr-config-file ../../test/defend_workflows_cypress/serverless_config",
     "cypress:dw:serverless:open": "yarn cypress:dw:serverless open",
     "cypress:dw:serverless:run": "yarn cypress:dw:serverless run",
+    "cypress:dw:mki:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel_serverless --config-file ./public/management/cypress/cypress_serverless_qa.config.ts",
+    "cypress:dw:qa:serverless:run": "yarn cypress:dw:mki:serverless run",
     "cypress:dw:serverless:changed-specs-only": "yarn cypress:dw:serverless run --changed-specs-only --env burn=2",
     "cypress:dw:endpoint": "echo '\n** WARNING **: Run script `cypress:dw:endpoint` no longer valid! Use `cypress:dw` instead\n'",
     "cypress:dw:endpoint:run": "echo '\n** WARNING **: Run script `cypress:dw:endpoint:run` no longer valid! Use `cypress:dw:run` instead\n'",

x-pack/plugins/security_solution/public/management/cypress/README.md

@@ -34,6 +34,7 @@ for more information.
 Similarly to Security Solution cypress tests, we use tags in order to select which tests we want to execute on which environment:
 
 - `@serverless` includes a test in the Serverless test suite. You need to explicitly add this tag to any test you want to run against a Serverless environment.
+- `'@cloudServerless` includes the test in the Serverless test suite that executes in a Cloud environment (MKI).
 - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
 - `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.
 - `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that we don't want to run the given test in Serverless.

x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts

@@ -74,6 +74,7 @@ export const getCypressBaseConfig = (
         experimentalRunAllSpecs: true,
         experimentalMemoryManagement: true,
         experimentalInteractiveRunEvents: true,
+        experimentalCspAllowList: ['default-src', 'script-src', 'script-src-elem'],
         setupNodeEvents: async (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => {
           // IMPORTANT: setting the log level should happen before any tooling is called
           setupToolingLogLevel(config);

x-pack/plugins/security_solution/public/management/cypress/cypress_serverless_qa.config.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineCypressConfig } from '@kbn/cypress-config';
+import { getCypressBaseConfig } from './cypress_base.config';
+
+// eslint-disable-next-line import/no-default-export
+export default defineCypressConfig(
+  getCypressBaseConfig({
+    env: {
+      IS_SERVERLESS: true,
+
+      // Uncomment to enable logging
+      // TOOLING_LOG_LEVEL: 'verbose',
+
+      grepTags: '@cloudServerless --@brokenInServerless',
+    },
+  })
+);

x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts

@@ -24,7 +24,7 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy';
 import { createEndpointHost } from '../../../tasks/create_endpoint_host';
 import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data';
 
-describe('Response console', { tags: ['@ess', '@serverless'] }, () => {
+describe('Response console', { tags: ['@ess', '@serverless', '@cloudServerless'] }, () => {
   beforeEach(() => {
     login();
   });

x-pack/plugins/security_solution/public/management/cypress/support/common.ts

@@ -25,7 +25,7 @@ export const setupStackServicesUsingCypressConfig = async (config: Cypress.Plugi
     password: config.env.KIBANA_PASSWORD,
     esUsername: config.env.ELASTICSEARCH_USERNAME,
     esPassword: config.env.ELASTICSEARCH_PASSWORD,
-    asSuperuser: true,
+    asSuperuser: !config.env.CLOUD_SERVERLESS,
   }).then(({ log, ...others }) => {
     return {
       ...others,

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -127,6 +127,7 @@ export const dataLoaders = (
 ): void => {
   // Env. variable is set by `cypress_serverless.config.ts`
   const isServerless = config.env.IS_SERVERLESS;
+  const isCloudServerless = Boolean(config.env.CLOUD_SERVERLESS);
   const stackServicesPromise = setupStackServicesUsingCypressConfig(config);
   const roleAndUserLoaderPromise: Promise<TestRoleAndUserLoader> = stackServicesPromise.then(
     ({ kbnClient, log }) => {
@@ -259,8 +260,8 @@ export const dataLoaders = (
     }: {
       endpointAgentIds: string[];
     }): Promise<DeleteAllEndpointDataResponse> => {
-      const { esClient } = await stackServicesPromise;
-      return deleteAllEndpointData(esClient, endpointAgentIds);
+      const { esClient, log } = await stackServicesPromise;
+      return deleteAllEndpointData(esClient, log, endpointAgentIds, !isCloudServerless);
     },
 
     /**

x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts

@@ -101,7 +101,7 @@ Cypress.Commands.add(
 
 Cypress.on('uncaught:exception', () => false);
 
-// Login as a SOC_MANAGER to properly initialize Security Solution App
+// Before any tests runs, Login and visit the Alerts page so that it properly initializes the Security Solution App
 before(() => {
   login(ROLE.soc_manager);
   loadPage('/app/security/alerts');

x-pack/plugins/security_solution/public/management/cypress/support/plugin_handlers/endpoint_data_loader.ts

@@ -10,6 +10,7 @@ import type { KbnClient } from '@kbn/test';
 import pRetry from 'p-retry';
 import { kibanaPackageJson } from '@kbn/repo-info';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { dump } from '../../../../../scripts/endpoint/common/utils';
 import { STARTED_TRANSFORM_STATES } from '../../../../../common/constants';
 import {
   ENDPOINT_ALERTS_INDEX,
@@ -79,8 +80,8 @@ export const cyLoadEndpointDataHandler = async (
   if (waitUntilTransformed) {
     // need this before indexing docs so that the united transform doesn't
     // create a checkpoint with a timestamp after the doc timestamps
-    await stopTransform(esClient, metadataTransformPrefix);
-    await stopTransform(esClient, METADATA_UNITED_TRANSFORM);
+    await stopTransform(esClient, log, metadataTransformPrefix);
+    await stopTransform(esClient, log, METADATA_UNITED_TRANSFORM);
   }
 
   // load data into the system
@@ -120,13 +121,23 @@ export const cyLoadEndpointDataHandler = async (
   return indexedData;
 };
 
-const stopTransform = async (esClient: Client, transformId: string): Promise<void> => {
-  await esClient.transform.stopTransform({
-    transform_id: `${transformId}*`,
-    force: true,
-    wait_for_completion: true,
-    allow_no_match: true,
-  });
+const stopTransform = async (
+  esClient: Client,
+  log: ToolingLog,
+  transformId: string
+): Promise<void> => {
+  await esClient.transform
+    .stopTransform({
+      transform_id: `${transformId}*`,
+      force: true,
+      wait_for_completion: true,
+      allow_no_match: true,
+    })
+    .catch((e) => {
+      Error.captureStackTrace(e);
+      log.verbose(dump(e, 8));
+      throw e;
+    });
 };
 
 const startTransform = async (esClient: Client, transformId: string): Promise<void> => {

x-pack/plugins/security_solution/scripts/endpoint/common/delete_all_endpoint_data.ts

@@ -7,7 +7,9 @@
 
 import type { Client, estypes } from '@elastic/elasticsearch';
 import assert from 'assert';
+import type { ToolingLog } from '@kbn/tooling-log';
 import { createEsClient, isServerlessKibanaFlavor } from './stack_services';
+import type { CreatedSecuritySuperuser } from './security_user_services';
 import { createSecuritySuperuser } from './security_user_services';
 
 export interface DeleteAllEndpointDataResponse {
@@ -22,24 +24,49 @@ export interface DeleteAllEndpointDataResponse {
  * **NOTE:** This utility will create a new role and user that has elevated privileges and access to system indexes.
  *
  * @param esClient
+ * @param log
  * @param endpointAgentIds
+ * @param asSuperuser
  */
 export const deleteAllEndpointData = async (
   esClient: Client,
-  endpointAgentIds: string[]
+  log: ToolingLog,
+  endpointAgentIds: string[],
+  /** If true, then a new user will be created that has full privileges to indexes (especially system indexes) */
+  asSuperuser: boolean = true
 ): Promise<DeleteAllEndpointDataResponse> => {
   assert(endpointAgentIds.length > 0, 'At least one endpoint agent id must be defined');
 
-  const isServerless = await isServerlessKibanaFlavor(esClient);
-  const unrestrictedUser = isServerless
-    ? { password: 'changeme', username: 'system_indices_superuser', created: false }
-    : await createSecuritySuperuser(esClient, 'super_superuser');
-  const esUrl = getEsUrlFromClient(esClient);
-  const esClientUnrestricted = createEsClient({
-    url: esUrl,
-    username: unrestrictedUser.username,
-    password: unrestrictedUser.password,
-  });
+  let esClientUnrestricted = esClient;
+
+  if (asSuperuser) {
+    log.debug(`Looking to use a superuser type of account`);
+
+    const isServerless = await isServerlessKibanaFlavor(esClient);
+    let unrestrictedUser: CreatedSecuritySuperuser | undefined;
+
+    if (isServerless) {
+      log.debug(`In serverless mode. Creating new ES Client using 'system_indices_superuser'`);
+
+      unrestrictedUser = {
+        password: 'changeme',
+        username: 'system_indices_superuser',
+        created: false,
+      };
+    } else {
+      log.debug(`Creating new superuser account [super_superuser]`);
+      unrestrictedUser = await createSecuritySuperuser(esClient, 'super_superuser');
+    }
+
+    if (unrestrictedUser) {
+      const esUrl = getEsUrlFromClient(esClient);
+      esClientUnrestricted = createEsClient({
+        url: esUrl,
+        username: unrestrictedUser.username,
+        password: unrestrictedUser.password,
+      });
+    }
+  }
 
   const queryString = endpointAgentIds.map((id) => `(${id})`).join(' OR ');
 
@@ -56,6 +83,8 @@ export const deleteAllEndpointData = async (
     conflicts: 'proceed',
   });
 
+  log.verbose(`All deleted documents:\n`, deleteResponse);
+
   return {
     count: deleteResponse.deleted ?? 0,
     query: queryString,

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts

@@ -42,7 +42,7 @@ import {
 import { maybeCreateDockerNetwork, SERVERLESS_NODES, verifyDockerInstalled } from '@kbn/es';
 import { resolve } from 'path';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { captureCallingStack, prefixedOutputLogger } from '../utils';
+import { captureCallingStack, dump, prefixedOutputLogger } from '../utils';
 import {
   createToolingLogger,
   RETRYABLE_TRANSIENT_ERRORS,
@@ -62,7 +62,6 @@ import {
   getFleetElasticsearchOutputHost,
   waitForHostToEnroll,
 } from '../fleet_services';
-import { dump } from '../../endpoint_agent_runner/utils';
 import { getLocalhostRealIp } from '../network_services';
 import { isLocalhost } from '../is_localhost';
 

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -350,12 +350,25 @@ export const fetchIntegrationPolicyList = async (
  * @param kbnClient
  */
 export const getAgentVersionMatchingCurrentStack = async (
-  kbnClient: KbnClient
+  kbnClient: KbnClient,
+  log: ToolingLog = createToolingLogger()
 ): Promise<string> => {
   const kbnStatus = await fetchKibanaStatus(kbnClient);
+
+  log.debug(`Kibana status:\n`, kbnStatus);
+
+  if (!kbnStatus.version) {
+    throw new Error(
+      `Kibana status api response did not include 'version' information - possibly due to invalid credentials`
+    );
+  }
+
   const agentVersions = await axios
     .get('https://artifacts-api.elastic.co/v1/versions')
-    .then((response) => map(response.data.versions, (version) => version.split('-SNAPSHOT')[0]));
+    .then((response) => {
+      log.verbose(`Agent Version:\n`, response.data);
+      return map(response.data.versions, (version) => version.split('-SNAPSHOT')[0]);
+    });
 
   let version =
     semver.maxSatisfying(agentVersions, `<=${kbnStatus.version.number}`) ??

x-pack/plugins/security_solution/scripts/endpoint/common/security_user_services.ts

@@ -8,11 +8,17 @@
 import type { Client } from '@elastic/elasticsearch';
 import { userInfo } from 'os';
 
+export interface CreatedSecuritySuperuser {
+  username: string;
+  password: string;
+  created: boolean;
+}
+
 export const createSecuritySuperuser = async (
   esClient: Client,
   username: string = userInfo().username,
   password: string = 'changeme'
-): Promise<{ username: string; password: string; created: boolean }> => {
+): Promise<CreatedSecuritySuperuser> => {
   if (!username || !password) {
     throw new Error(`username and password require values.`);
   }