[ResponseOps][Alerting] Do not return alerts from internally managed rule types

x-pack/platform/plugins/shared/alerting/README.md

@@ -14,24 +14,30 @@ Table of Contents
 	- [Terminology](#terminology)
 	- [Usage](#usage)
 	- [Alerting API Keys](#alerting-api-keys)
-	- [Plugin Status](#plugin-status)
 	- [Rule Types](#rule-types)
 		- [Methods](#methods)
-		- [Alerts as Data](#alerts-as-data)
 		- [Executor](#executor)
-		- [Action variables](#action-variables)
+		- [Alerts as Data](#alerts-as-data)
+		- [Action Variables](#action-variables)
+		- [useSavedObjectReferences Hooks](#usesavedobjectreferences-hooks)
 	- [Recovered Alerts](#recovered-alerts)
 	- [Licensing](#licensing)
 	- [Documentation](#documentation)
 	- [Tests](#tests)
 		- [Example](#example)
 	- [Role Based Access-Control](#role-based-access-control)
-	- [Alerting Navigation](#alert-navigation)
+		- [Subfeature privileges](#subfeature-privileges)
+		- [`read` privileges vs. `all` privileges](#read-privileges-vs-all-privileges)
+	- [Alert Navigation](#alert-navigation)
+		- [registerNavigation](#registernavigation)
+		- [registerDefaultNavigation](#registerdefaultnavigation)
+		- [Balancing both APIs side by side](#balancing-both-apis-side-by-side)
 	- [Internal HTTP APIs](#internal-http-apis)
 		- [`GET /internal/alerting/rule/{id}/state`: Get rule state](#get-internalalertingruleidstate-get-rule-state)
-		- [`GET /internal/alerting/rule/{id}/_alert_summary`: Get rule alert summary](#get-internalalertingruleidalertsummary-get-rule-alert-summary)
-		- [`POST /api/alerting/rule/{id}/_update_api_key`: Update rule API key](#post-internalalertingruleidupdateapikey-update-rule-api-key)
+		- [`GET /internal/alerting/rule/{id}/_alert_summary`: Get rule alert summary](#get-internalalertingruleid_alert_summary-get-rule-alert-summary)
+		- [`POST /api/alerting/rule/{id}/_update_api_key`: Update rule API key](#post-apialertingruleid_update_api_key-update-rule-api-key)
 	- [Alert Factory](#alert-factory)
+		- [When should I use `setContext`?](#when-should-i-use-setcontext)
 	- [Templating Actions](#templating-actions)
 		- [Examples](#examples)
 
@@ -102,6 +108,7 @@ The following table describes the properties of the `options` object.
 |alerts|(Optional) Specify options for writing alerts as data documents for this rule type. This feature is currently under development so this field is optional but we will eventually make this a requirement of all rule types. For full details, see the alerts as data section below.|IRuleTypeAlerts|
 |autoRecoverAlerts|(Optional) Whether the framework should determine if alerts have recovered between rule runs. If not specified, the default value of `true` is used. |boolean|
 |getViewInAppRelativeUrl|(Optional) When developing a rule type, you can choose to implement this hook for generating a link back to the Kibana application that can be used in alert actions. If not specified, a generic link back to the Rule Management app is generated.|Function|
+|internallyManaged|(Optional) Indicates that the rule type is managed internally by a Kibana plugin. Alerts of internally managed rule types are not returned by the APIs and thus not shown in the alerts table.|boolean|
 
 ### Executor
 

x-pack/platform/plugins/shared/alerting/server/rule_type_registry.ts

@@ -71,6 +71,7 @@ export interface RegistryRuleType
     | 'defaultScheduleInterval'
     | 'doesSetRecoveryContext'
     | 'alerts'
+    | 'internallyManaged'
   > {
   id: string;
   enabledInLicense: boolean;

x-pack/platform/plugins/shared/alerting/server/types.ts

@@ -350,6 +350,11 @@ export interface RuleType<
    */
   autoRecoverAlerts?: boolean;
   getViewInAppRelativeUrl?: GetViewInAppRelativeUrlFn<Params>;
+  /**
+   * Indicates that the rule type is managed internally by a Kibana plugin.
+   * Alerts of internally managed rule types are not returned by the APIs and thus not shown in the alerts table.
+   */
+  internallyManaged?: boolean;
 }
 export type UntypedRuleType = RuleType<
   RuleTypeParams,

x-pack/platform/plugins/shared/rule_registry/server/search_strategy/search_strategy.test.ts

@@ -819,4 +819,38 @@ describe('ruleRegistrySearchStrategyProvider()', () => {
       })
     );
   });
+
+  it('removes internally managed rule types', async () => {
+    const request: RuleRegistrySearchRequest = {
+      ruleTypeIds: ['.es-query', '.internally-managed', '.not-internally-managed'],
+      trackScores: true,
+    };
+
+    const options = {};
+    const deps = {
+      request: {},
+    };
+
+    getAuthorizedRuleTypesMock.mockResolvedValue([]);
+    getAlertIndicesAliasMock.mockReturnValue(['security-siem']);
+    alerting.listTypes.mockReturnValue(
+      // @ts-expect-error: rule type properties are not needed for the test
+      new Map([
+        ['.es-query', {}],
+        ['.internally-managed', { internallyManaged: true }],
+        ['.not-internally-managed', { internallyManaged: false }],
+      ])
+    );
+
+    const strategy = ruleRegistrySearchStrategyProvider(data, alerting, logger, security, spaces);
+
+    await lastValueFrom(
+      strategy.search(request, options, deps as unknown as SearchStrategyDependencies)
+    );
+
+    expect(authorizationMock.getAllAuthorizedRuleTypesFindOperation).toHaveBeenCalledWith({
+      authorizationEntity: 'alert',
+      ruleTypeIds: ['.es-query', '.not-internally-managed'],
+    });
+  });
 });

x-pack/platform/plugins/shared/rule_registry/server/search_strategy/search_strategy.ts

@@ -10,6 +10,7 @@ import { map, mergeMap, catchError, of } from 'rxjs';
 import type { estypes } from '@elastic/elasticsearch';
 import type { Logger } from '@kbn/core/server';
 import { from } from 'rxjs';
+import type { RegistryRuleType } from '@kbn/alerting-plugin/server/rule_type_registry';
 import { ENHANCED_ES_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
 import type { ISearchStrategy, PluginStart } from '@kbn/data-plugin/server';
 import type { AlertingServerStart } from '@kbn/alerting-plugin/server';
@@ -61,8 +62,11 @@ export const ruleRegistrySearchStrategyProvider = (
 
       const registeredRuleTypes = alerting.listTypes();
 
+      const ruleTypesWithoutInternalRuleTypes =
+        getRuleTypesWithoutInternalRuleTypes(registeredRuleTypes);
+
       const [validRuleTypeIds, _] = partition(request.ruleTypeIds, (ruleTypeId) =>
-        registeredRuleTypes.has(ruleTypeId)
+        ruleTypesWithoutInternalRuleTypes.has(ruleTypeId)
       );
 
       if (isAnyRuleTypeESAuthorized && !isEachRuleTypeESAuthorized) {
@@ -235,3 +239,11 @@ export const ruleRegistrySearchStrategyProvider = (
     },
   };
 };
+
+const getRuleTypesWithoutInternalRuleTypes = (registeredRuleTypes: Map<string, RegistryRuleType>) =>
+  new Map(
+    Array.from(registeredRuleTypes).filter(
+      ([_id, ruleType]) =>
+        ruleType.internallyManaged == null || !Boolean(ruleType.internallyManaged)
+    )
+  );

x-pack/platform/plugins/shared/streams/server/lib/rules/esql/register.ts

@@ -55,5 +55,6 @@ export function esqlRuleType(): PersistenceAlertType<
       shouldWrite: false,
       isSpaceAware: false,
     },
+    internallyManaged: true,
   };
 }

x-pack/platform/test/alerting_api_integration/common/lib/alert_utils.ts

@@ -754,3 +754,15 @@ function getAlwaysFiringRuleWithSystemAction(reference: string) {
     ],
   };
 }
+
+export function getAlwaysFiringInternalRule() {
+  return {
+    enabled: true,
+    name: 'Internal Rule',
+    schedule: { interval: '1m' },
+    tags: [],
+    rule_type_id: 'test.internal-rule-type',
+    consumer: 'alertsFixture',
+    params: {},
+  };
+}

x-pack/platform/test/alerting_api_integration/common/plugins/alerts/server/rule_types.ts

@@ -1192,6 +1192,41 @@ function getSeverityRuleType() {
   return result;
 }
 
+const getInternalRuleType = () => {
+  const result: RuleType<{}, never, {}, {}, {}, 'default'> = {
+    id: 'test.internal-rule-type',
+    name: 'Test: Internal Rule Type',
+    actionGroups: [{ id: 'default', name: 'Default' }],
+    validate: {
+      params: schema.any(),
+    },
+    category: 'management',
+    producer: 'alertsFixture',
+    solution: 'stack',
+    defaultActionGroupId: 'default',
+    minimumLicenseRequired: 'basic',
+    isExportable: true,
+    internallyManaged: true,
+    async executor(ruleExecutorOptions) {
+      const { services } = ruleExecutorOptions;
+
+      services.alertsClient?.report({ id: '1', actionGroup: 'default' });
+      services.alertsClient?.report({ id: '2', actionGroup: 'default' });
+
+      return { state: {} };
+    },
+    alerts: {
+      context: 'observability.test.alerts',
+      mappings: {
+        fieldMap: {},
+      },
+      useLegacyAlerts: true,
+      shouldWrite: true,
+    },
+  };
+  return result;
+};
+
 async function sendSignal(
   logger: Logger,
   es: ElasticsearchClient,
@@ -1531,4 +1566,5 @@ export function defineRuleTypes(
   alerting.registerType(getPatternFiringAlertsAsDataRuleType());
   alerting.registerType(getWaitingRuleType(logger));
   alerting.registerType(getSeverityRuleType());
+  alerting.registerType(getInternalRuleType());
 }

x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts

@@ -6,8 +6,13 @@
  */
 import expect from '@kbn/expect';
 
-import { ALERT_START } from '@kbn/rule-data-utils';
+import { ALERT_RULE_TYPE_ID, ALERT_START } from '@kbn/rule-data-utils';
 import type { RuleRegistrySearchResponse } from '@kbn/rule-registry-plugin/common';
+import { ObjectRemover } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib';
+import { getAlwaysFiringInternalRule } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib/alert_utils';
+import { getEventLog } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib';
+import type { RetryService } from '@kbn/ftr-common-functional-services';
+import type { Client } from '@elastic/elasticsearch';
 import type { FtrProviderContext } from '../../../common/ftr_provider_context';
 import {
   obsOnlySpacesAll,
@@ -28,6 +33,9 @@ export default ({ getService }: FtrProviderContext) => {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const secureSearch = getService('secureSearch');
   const kbnClient = getService('kibanaServer');
+  const es = getService('es');
+  const supertest = getService('supertest');
+  const retry = getService('retry');
 
   describe('ruleRegistryAlertsSearchStrategy', () => {
     let kibanaVersion: string;
@@ -983,6 +991,55 @@ export default ({ getService }: FtrProviderContext) => {
         expect(result.rawResponse.hits.total).to.eql(0);
       });
     });
+
+    describe('internal rule types', () => {
+      const alertAsDataIndex = '.internal.alerts-observability.test.alerts.alerts-default-000001';
+      const objectRemover = new ObjectRemover(supertest);
+      const rulePayload = getAlwaysFiringInternalRule();
+      let ruleId: string;
+
+      before(async () => {
+        await deleteAllAlertsFromIndex(alertAsDataIndex, es);
+      });
+
+      beforeEach(async () => {
+        const { body: createdRule1 } = await supertest
+          .post('/api/alerting/rule')
+          .set('kbn-xsrf', 'foo')
+          .send(rulePayload)
+          .expect(200);
+
+        ruleId = createdRule1.id;
+        objectRemover.add('default', createdRule1.id, 'rule', 'alerting');
+      });
+
+      afterEach(async () => {
+        await deleteAllAlertsFromIndex(alertAsDataIndex, es);
+        await objectRemover.removeAll();
+      });
+
+      it('should not return alerts from internal rule types', async () => {
+        await waitForRuleExecution(retry, getService, ruleId);
+        await waitForActiveAlerts(es, retry, alertAsDataIndex, rulePayload.rule_type_id);
+
+        const result = await secureSearch.send<RuleRegistrySearchResponse>({
+          supertestWithoutAuth,
+          auth: {
+            username: superUser.username,
+            password: superUser.password,
+          },
+          referer: 'test',
+          internalOrigin: 'Kibana',
+          options: {
+            ruleTypeIds: [rulePayload.rule_type_id],
+          },
+          strategy: 'privateRuleRegistryAlertsSearchStrategy',
+        });
+
+        expect(result.rawResponse.hits.total).to.eql(0);
+        expect(result.rawResponse.hits.hits.length).to.eql(0);
+      });
+    });
   });
 };
 
@@ -1001,3 +1058,51 @@ const validateRuleTypeIds = (result: RuleRegistrySearchResponse, ruleTypeIdsToVe
     )
   ).to.eql(true);
 };
+
+const waitForRuleExecution = async (
+  retry: RetryService,
+  getService: FtrProviderContext['getService'],
+  ruleId: string
+) => {
+  return await retry.try(async () => {
+    await getEventLog({
+      getService,
+      spaceId: 'default',
+      type: 'alert',
+      id: ruleId,
+      provider: 'alerting',
+      actions: new Map([['active-instance', { gte: 1 }]]),
+    });
+  });
+};
+
+const waitForActiveAlerts = async (
+  es: Client,
+  retry: RetryService,
+  alertAsDataIndex: string,
+  ruleTypeId: string
+) => {
+  await retry.try(async () => {
+    const {
+      hits: { hits: activeAlerts },
+    } = await es.search({
+      index: alertAsDataIndex,
+      query: { match_all: {} },
+    });
+
+    activeAlerts.forEach((activeAlert: any) => {
+      expect(activeAlert._source[ALERT_RULE_TYPE_ID]).eql(ruleTypeId);
+    });
+  });
+};
+
+const deleteAllAlertsFromIndex = async (index: string, es: Client) => {
+  await es.deleteByQuery({
+    index,
+    query: {
+      match_all: {},
+    },
+    conflicts: 'proceed',
+    ignore_unavailable: true,
+  });
+};