[Observability solution] [SLO] Run burn rate api tests in serverless & ess using mocha tagging

.buildkite/ftr_configs.yml

@@ -18,6 +18,8 @@ disabled:
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
   - x-pack/test/security_solution_endpoint/config.base.ts
   - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts
+  - x-pack/test/observability_solution_api_integration/config/ess/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -568,3 +570,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/serverless.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/ess.config.ts

.eslintrc.js

@@ -614,6 +614,8 @@ module.exports = {
         'x-pack/test/profiling_api_integration/**/*.ts',
         'x-pack/test/security_solution_api_integration/*/test_suites/**/*',
         'x-pack/test/security_solution_api_integration/**/config*.ts',
+        'x-pack/test/observability_solution_api_integration/*/test_suites/**/*',
+        'x-pack/test/observability_solution_api_integration/**/config*.ts',
       ],
       rules: {
         'import/no-default-export': 'off',
@@ -1847,6 +1849,14 @@ module.exports = {
       },
     },
 
+    /** Observerability Solution API Integration tests
+     * Ensures appropriate mocha tagging for tests
+     */
+    {
+      files: ['x-pack/test/observability_solution_api_integration/**/*.{ts,tsx}'],
+      rules: { '@kbn/eslint/require_mocha_tagging': 'error' },
+    },
+
     /**
      * Code inside .buildkite runs separately from everything else in CI, before bootstrap, with ts-node. It needs a few tweaks because of this.
      */

packages/kbn-eslint-plugin-eslint/index.js

@@ -18,5 +18,6 @@ module.exports = {
     no_constructor_args_in_property_initializers: require('./rules/no_constructor_args_in_property_initializers'),
     no_this_in_property_initializers: require('./rules/no_this_in_property_initializers'),
     no_unsafe_console: require('./rules/no_unsafe_console'),
+    require_mocha_tagging: require('./rules/require_mocha_tagging'),
   },
 };

packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.js

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const tsEstree = require('@typescript-eslint/typescript-estree');
+const esTypes = tsEstree.AST_NODE_TYPES;
+
+/** @typedef {import("eslint").Rule.RuleModule} Rule */
+/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.Node} Node */
+/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.CallExpression} CallExpression */
+/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.FunctionExpression} FunctionExpression */
+/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.ArrowFunctionExpression} ArrowFunctionExpression */
+/** @typedef {import("eslint").Rule.RuleFixer} Fixer */
+
+const ERROR_MSG = `Describe blocks must be tagged with either @ess or @serverless to specify the test execution environment. Ex: describe('@ess @serverless API Integration test', () => {})`;
+
+/**
+ * @param {any} context
+ * @param {CallExpression} node
+ */
+const isDescribeBlockWithoutTagging = (node) => {
+  const isDescribeBlock =
+    node.type === esTypes.CallExpression &&
+    node.callee.type === esTypes.Identifier &&
+    node.callee.name === 'describe' &&
+    node.arguments.length >= 1;
+
+  if (!isDescribeBlock) {
+    return false;
+  }
+  const title = node.arguments[0].value;
+  const hasTags = /^(@ess|@serverless)/.test(title);
+  if (hasTags) {
+    return false;
+  } else {
+    return true;
+  }
+};
+
+/** @type {Rule} */
+module.exports = {
+  meta: {
+    fixable: 'code',
+    schema: [],
+  },
+  create: (context) => ({
+    CallExpression(_) {
+      const node = /** @type {CallExpression} */ (_);
+
+      if (isDescribeBlockWithoutTagging(node)) {
+        context.report({
+          message: ERROR_MSG,
+          loc: node.arguments[0].loc,
+        });
+      }
+    },
+  }),
+};

packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.test.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { RuleTester } = require('eslint');
+const rule = require('./require_mocha_tagging');
+const dedent = require('dedent');
+
+const ruleTester = new RuleTester({
+  parser: require.resolve('@typescript-eslint/parser'),
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 2018,
+    ecmaFeatures: {
+      jsx: true,
+    },
+  },
+});
+
+ruleTester.run('@kbn/eslint/require_mocha_tagging', rule, {
+  valid: [
+    {
+      code: dedent`
+        describe('@ess @serverless API Integration test', () => {})
+      `,
+    },
+    {
+      code: dedent`
+        describe('@ess API Integration test', () => {})
+      `,
+    },
+    {
+      code: dedent`
+        describe('@serverless API Integration test', () => {})
+      `,
+    },
+  ],
+
+  invalid: [
+    {
+      code: dedent`
+        describe('API Integration test', () => {})
+      `,
+      errors: [
+        {
+          line: 1,
+          message:
+            'Passing an async function to .forEach() prevents promise rejections from being handled. Use asyncForEach() or similar helper from "@kbn/std" instead.',
+        },
+      ],
+    },
+  ],
+});

x-pack/test/api_integration/services/index.ts

@@ -24,6 +24,8 @@ import { IngestPipelinesProvider } from './ingest_pipelines';
 import { IndexManagementProvider } from './index_management';
 import { DataViewApiProvider } from './data_view_api';
 import { SloApiProvider } from './slo';
+import { SloApiProvider as SloApiProviderNew } from './slo_api';
+import { AlertingApiProvider } from './alerting_api';
 import { SecuritySolutionApiProvider } from './security_solution_api.gen';
 
 export const services = {
@@ -45,4 +47,6 @@ export const services = {
   indexManagement: IndexManagementProvider,
   slo: SloApiProvider,
   securitySolutionApi: SecuritySolutionApiProvider,
+  alertingApi: AlertingApiProvider,
+  sloApi: SloApiProviderNew, // TODO: Need to unify SloApiProvider and SloApiProviderNew, there was already an slo service here, part of the slo API migration issue https://github.com/elastic/kibana/issues/183397
 };

x-pack/test_serverless/api_integration/services/slo_api.ts → x-pack/test/api_integration/services/slo_api.ts

@@ -10,7 +10,7 @@ import {
   FetchHistoricalSummaryResponse,
 } from '@kbn/slo-schema';
 import * as t from 'io-ts';
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../functional/ftr_provider_context';
 
 type DurationUnit = 'm' | 'h' | 'd' | 'w' | 'M';
 

x-pack/test/observability_solution_api_integration/README.md

@@ -0,0 +1,78 @@
+# observability_solution_api_integration
+
+This directory serves as a centralized location to place the observability solution tests that run in Serverless and ESS environments
+
+## Subdirectories
+
+1. `config` stores base configurations specific to both the Serverless and ESS environments. These configurations build upon the base configuration provided by `x-pack/test_serverless` and `x-pack/test/api_integration`, incorporating additional settings such as environmental variables and tagging options
+
+2. `test_suites` directory houses all the tests along with the utility functions.
+
+## Overview
+
+- In this directory Mocha tagging is utilized to assign tags to specific test suites and individual test cases. This tagging system enables the ability to selectively apply tags to test suites and test cases, facilitating the exclusion of specific test cases within a test suite as needed.
+
+- Test suites and cases are prefixed with specific tags to determine their execution in particular environments or to exclude them from specific environments.
+
+- We are using the following tags:
+    * `@ess`: Runs in an ESS environment (on-prem installation) as part of the CI validation on PRs.
+
+    * `@serverless`: Runs in the first quality gate and in the periodic pipeline.
+
+    * `@skipInEss`: Skipped for ESS environment.
+
+    * `@skipInServerless`: Skipped for all quality gates, CI and periodic pipeline.
+
+ex:
+```
+ describe('@serverless @ess create_rules', () => { ==> tests in this suite will run in both Ess and Serverless
+   describe('creating rules', () => {}); 
+
+   describe('@skipInServerless missing timestamps', () => {}); ==> tests in this suite will be excluded in Serverless
+
+```
+
+# Adding new observabiluty area's tests
+
+1. Within the `test_suites` directory, create a new area folder, for example slos, rules, apm etc
+2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface.
+3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings.
+4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline.
+
+
+# Testing locally
+
+In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline.
+
+# How to run
+You can run various commands with different parameters for the different test worflows.
+
+The command structure follows this pattern:
+
+- `<test>`: The test workflow you want to run.
+- `<type>`: The type of operation, either "server" or "runner."
+- `<environment>`: The testing environment, such as "serverless," or "ess", specifies the correct configuration file for the tests.
+
+Run the server for "alerting_burn_rate" in the "serverless" environment:
+
+```shell
+npm run alerting_burn_rate:server:serverless
+```
+
+Run tests for "alerting_burn_rate" in the "serverless" environment:
+
+```shell
+npm run alerting_burn_rate:runner:serverless
+```
+
+Run the server for "alerting_burn_rate" in the "ess" environment:
+
+```shell
+npm run alerting_burn_rate:server:ess
+```
+
+Run tests for "alerting_burn_rate" in the "ess" environment:
+
+```shell
+npm run alerting_burn_rate:runner:ess
+```

x-pack/test/observability_solution_api_integration/config/ess/config.base.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../../api_integration/services';
+
+export interface CreateTestConfigOptions {
+  testFiles: string[];
+  junit: { reportName: string };
+  publicBaseUrl?: boolean;
+}
+
+export function createTestConfig(options: CreateTestConfigOptions) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const xPackApiIntegrationTestsConfig = await readConfigFile(
+      require.resolve('../../../api_integration/config.ts')
+    );
+
+    return {
+      ...xPackApiIntegrationTestsConfig.getAll(),
+      testFiles: options.testFiles,
+      services: {
+        ...services,
+      },
+      junit: {
+        reportName: 'X-Pack Οbservability Solution API Integration Tests',
+      },
+      mochaOpts: {
+        grep: '/^(?!.*@skipInEss).*@ess.*/',
+      },
+      kbnTestServer: {
+        ...xPackApiIntegrationTestsConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'),
+          ...(options.publicBaseUrl ? ['--server.publicBaseUrl=http://localhost:5620'] : []),
+        ],
+      },
+    };
+  };
+}

x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../../../test_serverless/api_integration/services';
+
+export interface CreateTestConfigOptions {
+  testFiles: string[];
+  junit: { reportName: string };
+}
+
+export function createTestConfig(options: CreateTestConfigOptions) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const svlSharedConfig = await readConfigFile(
+      require.resolve('../../../../test_serverless/shared/config.base.ts')
+    );
+
+    return {
+      ...svlSharedConfig.getAll(),
+      services: {
+        ...services,
+      },
+      kbnTestServer: {
+        ...svlSharedConfig.get('kbnTestServer'),
+        serverArgs: [...svlSharedConfig.get('kbnTestServer.serverArgs'), `--serverless=oblt`],
+      },
+      testFiles: options.testFiles,
+      junit: options.junit,
+      mochaOpts: {
+        ...svlSharedConfig.get('mochaOpts'),
+        grep: '/^(?!.*@skipInServerless).*@serverless.*/',
+      },
+    };
+  };
+}

x-pack/test/observability_solution_api_integration/ftr_provider_context.d.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { FtrProviderContext } from '../api_integration/ftr_provider_context';
+
+export type { FtrProviderContext };

x-pack/test/observability_solution_api_integration/package.json

@@ -0,0 +1,17 @@
+{
+  "author": "Elastic",
+  "name": "@kbn/observability_solution_api_integration",
+  "version": "1.0.0",
+  "private": true,
+  "license": "Elastic License 2.0",
+  "scripts": {
+    "alerting_burn_rate:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/serverless.config.ts",
+    "alerting_burn_rate:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/serverless.config.ts --grep @serverless --grep  @skipInServerless --invert",
+    "alerting_burn_rate:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/ess.config.ts",
+    "alerting_burn_rate:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/ess.config.ts --grep @ess --grep  @skipInEss --invert",
+    "alerting_custom_threshold:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/custom_threshold/configs/serverless.config.ts",
+    "alerting_custom_threshold:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/custom_threshold/configs/serverless.config.ts --grep @serverless --grep  @skipInServerless --invert",
+    "alerting_custom_threshold:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/custom_threshold/configs/ess.config.ts",
+    "alerting_custom_threshold:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/custom_threshold/configs/ess.config.ts --grep @ess --grep  @skipInEss --invert"
+  }
+}