[FTR] Split config listing for serverless projects

.buildkite/ftr_configs.yml

@@ -90,7 +90,8 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
 
 defaultQueue: 'n2-4-spot'
-enabled:
+
+enabled_stateful:
   - test/accessibility/config.ts
   - test/analytics/config.ts
   - test/api_integration/config.js
@@ -382,11 +383,8 @@ enabled:
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
-  - x-pack/test/security_solution_endpoint_api_int/serverless.config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
-  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts
-  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -398,34 +396,6 @@ enabled:
   - x-pack/test/ui_capabilities/spaces_only/config.ts
   - x-pack/test/upgrade_assistant_integration/config.js
   - x-pack/test/usage_collection/config.ts
-  - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
-  - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
-  - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
-  - x-pack/test_serverless/api_integration/test_suites/search/config.ts
-  - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts
-  - x-pack/test_serverless/api_integration/test_suites/security/config.ts
-  - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
-  - x-pack/test_serverless/functional/test_suites/observability/config.ts
-  - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
-  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts
-  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts
-  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts
-  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group4.ts
-  - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
-  - x-pack/test_serverless/functional/test_suites/search/config.ts
-  - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
-  - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
-  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts
-  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts
-  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts
-  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
-  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
-  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts
-  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts
-  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts
   - x-pack/performance/journeys/ecommerce_dashboard.ts
   - x-pack/performance/journeys/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys/flight_dashboard.ts
@@ -446,27 +416,64 @@ enabled:
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
 
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/ess.config.ts
+
+enabled_serverless_search:
+  - x-pack/test_serverless/api_integration/test_suites/search/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
+
+enabled_serverless_security:
+  - x-pack/test_serverless/api_integration/test_suites/security/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts
+  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
+  - x-pack/test/security_solution_endpoint_api_int/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/serverless.config.ts
+
+enabled_serverless_observability:
+  - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group4.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -112,42 +112,57 @@ function getEnabledFtrConfigs(patterns?: string[]) {
     if (!isObj(configs)) {
       throw new Error('expected yaml file to parse to an object');
     }
-    if (!configs.enabled) {
-      throw new Error('expected yaml file to have an "enabled" key');
-    }
-    if (
-      !Array.isArray(configs.enabled) ||
-      !configs.enabled.every(
-        (p): p is string | { [configPath: string]: { queue: string } } =>
-          typeof p === 'string' ||
-          (isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string'))
-      )
-    ) {
-      throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n
-  - {configPath}:
-      queue: {queueName}`);
-    }
+
     if (typeof configs.defaultQueue !== 'string') {
       throw new Error('expected yaml file to have a string "defaultQueue" key');
     }
-
     const defaultQueue = configs.defaultQueue;
     const ftrConfigsByQueue = new Map<string, string[]>();
-    for (const enabled of configs.enabled) {
-      const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0];
-      const queue = isObj(enabled) ? enabled[path].queue : defaultQueue;
 
-      if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) {
-        continue;
+    const enabledKeys = [
+      'enabled_stateful',
+      'enabled_serverless_search',
+      'enabled_serverless_security',
+      'enabled_serverless_observability',
+    ];
+    enabledKeys.forEach((key) => {
+      const enabledEntries = configs[key];
+      if (!enabledEntries) {
+        throw new Error(`expected yaml file to have an ${key} key`);
+      }
+      if (
+        !Array.isArray(enabledEntries) ||
+        !enabledEntries.every(
+          (p): p is string | { [configPath: string]: { queue: string } } =>
+            typeof p === 'string' ||
+            (isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string'))
+        )
+      ) {
+        throw new Error(`expected ${key} value to be an array of strings or objects shaped as:\n
+    - {configPath}:
+        queue: {queueName}`);
       }
 
-      const group = ftrConfigsByQueue.get(queue);
-      if (group) {
-        group.push(path);
-      } else {
-        ftrConfigsByQueue.set(queue, [path]);
+      if (typeof configs.defaultQueue !== 'string') {
+        throw new Error('expected yaml file to have a string "defaultQueue" key');
       }
-    }
+
+      for (const enabled of enabledEntries) {
+        const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0];
+        const queue = isObj(enabled) ? enabled[path].queue : defaultQueue;
+
+        if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) {
+          continue;
+        }
+
+        const group = ftrConfigsByQueue.get(queue);
+        if (group) {
+          group.push(path);
+        } else {
+          ftrConfigsByQueue.set(queue, [path]);
+        }
+      }
+    });
 
     return { defaultQueue, ftrConfigsByQueue };
   } catch (_) {

packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts

@@ -23,15 +23,21 @@ interface FtrConfigWithOptions {
 interface FtrConfigsManifest {
   defaultQueue: string;
   disabled: string[];
-  enabled: Array<string | FtrConfigWithOptions>;
+  enabled_stateful: Array<string | FtrConfigWithOptions>;
+  enabled_serverless_search: Array<string | FtrConfigWithOptions>;
+  enabled_serverless_security: Array<string | FtrConfigWithOptions>;
+  enabled_serverless_observability: Array<string | FtrConfigWithOptions>;
 }
 
 const ftrConfigsManifest: FtrConfigsManifest = JsYaml.safeLoad(
   Fs.readFileSync(Path.resolve(REPO_ROOT, FTR_CONFIGS_MANIFEST_REL), 'utf8')
 );
 
 export const FTR_CONFIGS_MANIFEST_PATHS = [
-  Object.values(ftrConfigsManifest.enabled),
+  Object.values(ftrConfigsManifest.enabled_stateful),
+  Object.values(ftrConfigsManifest.enabled_serverless_search),
+  Object.values(ftrConfigsManifest.enabled_serverless_security),
+  Object.values(ftrConfigsManifest.enabled_serverless_observability),
   Object.values(ftrConfigsManifest.disabled),
 ]
   .flat()

scripts/enabled_ftr_configs.js

@@ -12,7 +12,14 @@ var yaml = require('js-yaml');
 var fs = require('fs');
 
 try {
-  yaml.load(fs.readFileSync('.buildkite/ftr_configs.yml', 'utf8')).enabled.forEach(function (x) {
+  var configs = yaml.load(fs.readFileSync('.buildkite/ftr_configs.yml', 'utf8'));
+  var enabledConfigs = [].concat(
+    configs.enabled_stateful,
+    configs.enabled_serverless_search,
+    configs.enabled_serverless_security,
+    configs.enabled_serverless_observability
+  );
+  enabledConfigs.forEach(function (x) {
     console.log(x);
   });
 } catch (e) {

x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.tsx

@@ -7,13 +7,13 @@
 
 import React, { useRef } from 'react';
 import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner, useEuiTheme } from '@elastic/eui';
-import { first } from 'lodash';
 import { findInventoryModel } from '@kbn/metrics-data-access-plugin/common';
 import {
   InventoryItemType,
   SnapshotMetricType,
   SnapshotMetricTypeRT,
 } from '@kbn/metrics-data-access-plugin/common';
+import { first } from 'lodash';
 import { getCustomMetricLabel } from '../../../../../../common/formatters/get_custom_metric_label';
 import { SnapshotCustomMetricInput } from '../../../../../../common/http_api';
 import { useSourceContext } from '../../../../../containers/metrics_source';

x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_timeline.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { first } from 'lodash';
 import { useEffect, useMemo } from 'react';
 import type { InventoryItemType, SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
+import { first } from 'lodash';
 import { getIntervalInSeconds } from '../../../../../common/utils/get_interval_in_seconds';
 import { InfraTimerangeInput } from '../../../../../common/http_api/snapshot_api';
 import { useSnapshot } from './use_snaphot';

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts

@@ -7,7 +7,6 @@
 
 import { i18n } from '@kbn/i18n';
 import { ALERT_REASON, ALERT_ACTION_GROUP, ALERT_EVALUATION_VALUES } from '@kbn/rule-data-utils';
-import { first, get } from 'lodash';
 import {
   ActionGroup,
   ActionGroupIdsOf,
@@ -17,6 +16,7 @@ import {
 import { Alert, RuleTypeState } from '@kbn/alerting-plugin/server';
 import { getAlertUrl } from '@kbn/observability-plugin/common';
 import { SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
+import { first, get } from 'lodash';
 import { getOriginalActionGroup } from '../../../utils/get_original_action_group';
 import { AlertStates, InventoryMetricThresholdParams } from '../../../../common/alerting/metrics';
 import { createFormatter } from '../../../../common/formatters';

x-pack/test/security_solution_api_integration/README.md

@@ -33,7 +33,7 @@ ex:
 1. Within the `test_suites` directory, create a new area folder.
 2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface.
 3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings.
-4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline.
+4. Append a new entry in the `ftr_configs.yml` file (stateful configs under `enabled_stateful` and serverless configs under `enabled_serverless_security`) to enable the execution of the newly added tests within the CI pipeline.
 
 
 # Testing locally