[RAM] Mark disabled alerts as Untracked in both Stack and o11y

packages/kbn-alerts-ui-shared/src/alert_lifecycle_status_badge/index.tsx

@@ -9,7 +9,7 @@
 import React, { memo } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiBadge, EuiBadgeProps } from '@elastic/eui';
-import { AlertStatus, ALERT_STATUS_RECOVERED } from '@kbn/rule-data-utils';
+import { AlertStatus, ALERT_STATUS_RECOVERED, ALERT_STATUS_UNTRACKED } from '@kbn/rule-data-utils';
 
 export interface AlertLifecycleStatusBadgeProps {
   alertStatus: AlertStatus;
@@ -37,15 +37,27 @@ const FLAPPING_LABEL = i18n.translate(
   }
 );
 
+const UNTRACKED_LABEL = i18n.translate(
+  'alertsUIShared.components.alertLifecycleStatusBadge.untrackedLabel',
+  {
+    defaultMessage: 'Untracked',
+  }
+);
+
 interface BadgeProps {
   label: string;
   color: string;
+  isDisabled?: boolean;
   iconProps?: {
     iconType: EuiBadgeProps['iconType'];
   };
 }
 
 const getBadgeProps = (alertStatus: AlertStatus, flapping: boolean | undefined): BadgeProps => {
+  if (alertStatus === ALERT_STATUS_UNTRACKED) {
+    return { label: UNTRACKED_LABEL, color: 'default', isDisabled: true };
+  }
+
   // Prefer recovered over flapping
   if (alertStatus === ALERT_STATUS_RECOVERED) {
     return {
@@ -82,10 +94,15 @@ export const AlertLifecycleStatusBadge = memo((props: AlertLifecycleStatusBadgeP
 
   const castedFlapping = castFlapping(flapping);
 
-  const { label, color, iconProps } = getBadgeProps(alertStatus, castedFlapping);
+  const { label, color, iconProps, isDisabled } = getBadgeProps(alertStatus, castedFlapping);
 
   return (
-    <EuiBadge data-test-subj="alertLifecycleStatusBadge" color={color} {...iconProps}>
+    <EuiBadge
+      data-test-subj="alertLifecycleStatusBadge"
+      isDisabled={isDisabled}
+      color={color}
+      {...iconProps}
+    >
       {label}
     </EuiBadge>
   );

packages/kbn-rule-data-utils/src/alerts_as_data_status.ts

@@ -8,5 +8,9 @@
 
 export const ALERT_STATUS_ACTIVE = 'active';
 export const ALERT_STATUS_RECOVERED = 'recovered';
+export const ALERT_STATUS_UNTRACKED = 'untracked';
 
-export type AlertStatus = typeof ALERT_STATUS_ACTIVE | typeof ALERT_STATUS_RECOVERED;
+export type AlertStatus =
+  | typeof ALERT_STATUS_ACTIVE
+  | typeof ALERT_STATUS_RECOVERED
+  | typeof ALERT_STATUS_UNTRACKED;

x-pack/plugins/alerting/common/alert_summary.ts

@@ -40,4 +40,5 @@ export interface AlertStatus {
   activeStartDate?: string;
   flapping: boolean;
   maintenanceWindowIds?: string[];
+  tracked: boolean;
 }

x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts

@@ -3026,6 +3026,53 @@ describe('Alerts Client', () => {
           expect(recoveredAlert.hit).toBeUndefined();
         });
       });
+
+      describe('setAlertStatusToUntracked()', () => {
+        test('should call updateByQuery on provided ruleIds', async () => {
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>(
+            alertsClientParams
+          );
+
+          const opts = {
+            maxAlerts,
+            ruleLabel: `test: rule-name`,
+            flappingSettings: DEFAULT_FLAPPING_SETTINGS,
+            activeAlertsFromState: {},
+            recoveredAlertsFromState: {},
+          };
+          await alertsClient.initializeExecution(opts);
+
+          await alertsClient.setAlertStatusToUntracked(['test-index'], ['test-rule']);
+
+          expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(1);
+        });
+
+        test('should retry updateByQuery on failure', async () => {
+          clusterClient.updateByQuery.mockResponseOnce({
+            total: 10,
+            updated: 8,
+          });
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>(
+            alertsClientParams
+          );
+
+          const opts = {
+            maxAlerts,
+            ruleLabel: `test: rule-name`,
+            flappingSettings: DEFAULT_FLAPPING_SETTINGS,
+            activeAlertsFromState: {},
+            recoveredAlertsFromState: {},
+          };
+          await alertsClient.initializeExecution(opts);
+
+          await alertsClient.setAlertStatusToUntracked(['test-index'], ['test-rule']);
+
+          expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(2);
+          expect(logger.warn).toHaveBeenCalledWith(
+            'Attempt 1: Failed to untrack 2 of 10; indices test-index, ruleIds test-rule'
+          );
+        });
+      });
     });
   }
 });

x-pack/plugins/alerting/server/alerts_client/alerts_client.ts

@@ -6,7 +6,13 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { ALERT_RULE_UUID, ALERT_UUID } from '@kbn/rule-data-utils';
+import {
+  ALERT_RULE_UUID,
+  ALERT_STATUS,
+  ALERT_STATUS_UNTRACKED,
+  ALERT_STATUS_ACTIVE,
+  ALERT_UUID,
+} from '@kbn/rule-data-utils';
 import { chunk, flatMap, isEmpty, keys } from 'lodash';
 import { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { Alert } from '@kbn/alerts-as-data-utils';
@@ -198,6 +204,51 @@ export class AlertsClient<
     return { hits, total };
   }
 
+  public async setAlertStatusToUntracked(indices: string[], ruleIds: string[]) {
+    const esClient = await this.options.elasticsearchClientPromise;
+    const terms: Array<{ term: Record<string, { value: string }> }> = ruleIds.map((ruleId) => ({
+      term: {
+        [ALERT_RULE_UUID]: { value: ruleId },
+      },
+    }));
+    terms.push({
+      term: {
+        [ALERT_STATUS]: { value: ALERT_STATUS_ACTIVE },
+      },
+    });
+
+    try {
+      // Retry this updateByQuery up to 3 times to make sure the number of documents
+      // updated equals the number of documents matched
+      for (let retryCount = 0; retryCount < 3; retryCount++) {
+        const response = await esClient.updateByQuery({
+          index: indices,
+          allow_no_indices: true,
+          body: {
+            conflicts: 'proceed',
+            script: {
+              source: UNTRACK_UPDATE_PAINLESS_SCRIPT,
+              lang: 'painless',
+            },
+            query: {
+              bool: {
+                must: terms,
+              },
+            },
+          },
+        });
+        if (response.total === response.updated) break;
+        this.options.logger.warn(
+          `Attempt ${retryCount + 1}: Failed to untrack ${
+            (response.total ?? 0) - (response.updated ?? 0)
+          } of ${response.total}; indices ${indices}, ruleIds ${ruleIds}`
+        );
+      }
+    } catch (err) {
+      this.options.logger.error(`Error marking ${ruleIds} as untracked - ${err.message}`);
+    }
+  }
+
   public report(
     alert: ReportedAlert<
       AlertData,
@@ -562,3 +613,11 @@ export class AlertsClient<
     return this._isUsingDataStreams;
   }
 }
+
+const UNTRACK_UPDATE_PAINLESS_SCRIPT = `
+// Certain rule types don't flatten their AAD values, apply the ALERT_STATUS key to them directly
+if (!ctx._source.containsKey('${ALERT_STATUS}') || ctx._source['${ALERT_STATUS}'].empty) {
+  ctx._source.${ALERT_STATUS} = '${ALERT_STATUS_UNTRACKED}';
+} else {
+  ctx._source['${ALERT_STATUS}'] = '${ALERT_STATUS_UNTRACKED}'
+}`;

x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts

@@ -232,4 +232,8 @@ export class LegacyAlertsClient<
   }
 
   public async persistAlerts() {}
+
+  public async setAlertStatusToUntracked() {
+    return;
+  }
 }

x-pack/plugins/alerting/server/alerts_client/types.ts

@@ -63,6 +63,7 @@ export interface IAlertsClient<
     alertsToReturn: Record<string, RawAlertInstance>;
     recoveredAlertsToReturn: Record<string, RawAlertInstance>;
   };
+  setAlertStatusToUntracked(indices: string[], ruleIds: string[]): Promise<void>;
   factory(): PublicAlertFactory<
     State,
     Context,

x-pack/plugins/alerting/server/application/rule/methods/aggregate/aggregate_rules.test.ts

@@ -56,6 +56,8 @@ const rulesClientParams: jest.Mocked<ConstructorOptions> = {
   kibanaVersion,
   isAuthenticationTypeAPIKey: jest.fn(),
   getAuthenticationAPIKey: jest.fn(),
+  getAlertIndicesAlias: jest.fn(),
+  alertsService: null,
   maxScheduledPerMinute: 1000,
   internalSavedObjectsRepository,
 };

x-pack/plugins/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.test.ts

@@ -79,6 +79,8 @@ const rulesClientParams: jest.Mocked<ConstructorOptions> = {
   minimumScheduleInterval: { value: '1m', enforce: false },
   isAuthenticationTypeAPIKey: jest.fn(),
   getAuthenticationAPIKey: jest.fn(),
+  getAlertIndicesAlias: jest.fn(),
+  alertsService: null,
 };
 
 const getBulkOperationStatusErrorResponse = (statusCode: number) => ({

x-pack/plugins/alerting/server/application/rule/methods/bulk_edit/bulk_edit_rules.test.ts

@@ -102,6 +102,8 @@ const rulesClientParams: jest.Mocked<ConstructorOptions> = {
   minimumScheduleInterval: { value: '1m', enforce: false },
   isAuthenticationTypeAPIKey: isAuthenticationTypeApiKeyMock,
   getAuthenticationAPIKey: getAuthenticationApiKeyMock,
+  getAlertIndicesAlias: jest.fn(),
+  alertsService: null,
 };
 const paramsModifier = jest.fn();
 

x-pack/plugins/alerting/server/application/rule/methods/create/create_rule.test.ts

@@ -82,6 +82,8 @@ const rulesClientParams: jest.Mocked<ConstructorOptions> = {
   minimumScheduleInterval: { value: '1m', enforce: false },
   isAuthenticationTypeAPIKey: jest.fn(),
   getAuthenticationAPIKey: jest.fn(),
+  getAlertIndicesAlias: jest.fn(),
+  alertsService: null,
 };
 
 beforeEach(() => {

x-pack/plugins/alerting/server/application/rule/methods/get_schedule_frequency/get_schedule_frequency.test.ts

@@ -53,6 +53,8 @@ const rulesClientParams: jest.Mocked<ConstructorOptions> = {
   minimumScheduleInterval: { value: '1m', enforce: false },
   isAuthenticationTypeAPIKey: jest.fn(),
   getAuthenticationAPIKey: jest.fn(),
+  getAlertIndicesAlias: jest.fn(),
+  alertsService: null,
 };
 
 const getMockAggregationResult = (

x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts

@@ -127,6 +127,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": true,
             "status": "OK",
+            "tracked": true,
             "uuid": undefined,
           },
           "alert-2": Object {
@@ -135,6 +136,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": true,
             "status": "OK",
+            "tracked": true,
             "uuid": undefined,
           },
         },
@@ -241,6 +243,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "OK",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -283,6 +286,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "OK",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -324,6 +328,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "OK",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -366,6 +371,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -408,6 +414,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -450,6 +457,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -490,6 +498,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },
@@ -534,6 +543,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": true,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
           "alert-2": Object {
@@ -542,6 +552,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": true,
             "status": "OK",
+            "tracked": true,
             "uuid": "uuid-2",
           },
         },
@@ -593,6 +604,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
           "alert-2": Object {
@@ -601,6 +613,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": false,
             "muted": false,
             "status": "OK",
+            "tracked": true,
             "uuid": "uuid-2",
           },
         },
@@ -639,6 +652,7 @@ describe('alertSummaryFromEventLog', () => {
             "flapping": true,
             "muted": false,
             "status": "Active",
+            "tracked": true,
             "uuid": "uuid-1",
           },
         },

x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.ts

@@ -105,6 +105,8 @@ export function alertSummaryFromEventLog(params: AlertSummaryFromEventLogParams)
         status.activeStartDate = undefined;
         status.actionGroupId = undefined;
     }
+
+    status.tracked = action !== EVENT_LOG_ACTIONS.untrackedInstance;
   }
 
   for (const event of executionEvents.reverse()) {
@@ -169,6 +171,7 @@ function getAlertStatus(
     actionGroupId: undefined,
     activeStartDate: undefined,
     flapping: false,
+    tracked: true,
   };
   alerts.set(alertId, status);
   return status;