[Response Ops][Alerting] Remove echoed field value from bulk error responses when indexing alerts

x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts

@@ -1133,6 +1133,22 @@ describe('Alerts Client', () => {
                   },
                 },
               },
+              {
+                index: {
+                  _index: '.internal.alerts-test.alerts-default-000001',
+                  _id: '7',
+                  status: 404,
+                  error: {
+                    type: 'mapper_parsing_exception',
+                    reason:
+                      "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'",
+                    caused_by: {
+                      type: 'illegal_state_exception',
+                      reason: "Can't get text on a START_OBJECT at 1:3845",
+                    },
+                  },
+                },
+              },
               {
                 index: {
                   _index: '.internal.alerts-test.alerts-default-000002',
@@ -1164,7 +1180,7 @@ describe('Alerts Client', () => {
 
           expect(clusterClient.bulk).toHaveBeenCalled();
           expect(logger.error).toHaveBeenCalledWith(
-            `Error writing alerts: 1 successful, 0 conflicts, 1 errors: Validation Failed: 1: index is missing;2: type is missing;`
+            `Error writing alerts: 1 successful, 0 conflicts, 2 errors: Validation Failed: 1: index is missing;2: type is missing;; failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.`
           );
         });
 

x-pack/plugins/alerting/server/alerts_client/index.ts

@@ -8,3 +8,4 @@
 export { type LegacyAlertsClientParams, LegacyAlertsClient } from './legacy_alerts_client';
 export { AlertsClient } from './alerts_client';
 export type { AlertRuleData } from './types';
+export { sanitizeBulkErrorResponse } from './lib';

x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts

@@ -23,6 +23,7 @@ import {
 } from '@kbn/rule-data-utils';
 
 import { zip, get } from 'lodash';
+import { sanitizeBulkErrorResponse } from '../..';
 
 // these fields are the one's we'll refresh from the fresh mget'd docs
 const REFRESH_FIELDS_ALWAYS = [ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, ALERT_CASE_IDS];
@@ -269,8 +270,9 @@ interface ResponseStatsResult {
 
 // generate a summary of the original bulk request attempt, for logging
 function getResponseStats(bulkResponse: BulkResponse): ResponseStatsResult {
+  const sanitizedResponse = sanitizeBulkErrorResponse(bulkResponse) as BulkResponse;
   const stats: ResponseStatsResult = { success: 0, conflicts: 0, errors: 0, messages: [] };
-  for (const item of bulkResponse.items) {
+  for (const item of sanitizedResponse.items) {
     const op = item.create || item.index || item.update || item.delete;
     if (op?.error) {
       if (op?.status === 409 && op === item.index) {

x-pack/plugins/alerting/server/alerts_client/lib/index.ts

@@ -16,3 +16,4 @@ export {
   getContinualAlertsQuery,
 } from './get_summarized_alerts_query';
 export { expandFlattenedAlert } from './format_alert';
+export { sanitizeBulkErrorResponse } from './sanitize_bulk_response';

x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts

@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { TransportResult } from '@elastic/elasticsearch';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { sanitizeBulkErrorResponse } from './sanitize_bulk_response';
+
+// Using https://www.elastic.co/guide/en/elasticsearch/reference/8.11/docs-bulk.html
+describe('sanitizeBulkErrorResponse', () => {
+  test('should not modify success response', () => {
+    const responseBody = {
+      errors: false,
+      took: 1,
+      items: [
+        {
+          index: {
+            _index: 'test',
+            _id: '1',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 0,
+            _primary_term: 1,
+          },
+        },
+        {
+          delete: {
+            _index: 'test',
+            _id: '2',
+            _version: 1,
+            result: 'not_found',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 404,
+            _seq_no: 1,
+            _primary_term: 2,
+          },
+        },
+        {
+          create: {
+            _index: 'test',
+            _id: '3',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 2,
+            _primary_term: 3,
+          },
+        },
+        {
+          update: {
+            _index: 'test',
+            _id: '1',
+            _version: 2,
+            result: 'updated',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 200,
+            _seq_no: 3,
+            _primary_term: 4,
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody);
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody);
+  });
+
+  test('should not modify error response without field preview', () => {
+    const responseBody = {
+      took: 486,
+      errors: true,
+      items: [
+        {
+          update: {
+            _index: 'index1',
+            _id: '5',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[5]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          delete: {
+            _index: 'index1',
+            _id: '6',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[6]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          create: {
+            _index: 'test',
+            _id: '3',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 2,
+            _primary_term: 3,
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody);
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody);
+  });
+
+  test('should sanitize error response with field preview', () => {
+    const responseBody = {
+      took: 486,
+      errors: true,
+      items: [
+        {
+          update: {
+            _index: 'index1',
+            _id: '5',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[5]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          update: {
+            _index: 'index1',
+            _id: '6',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[6]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          create: {
+            _index: 'index1',
+            _id: '7',
+            status: 404,
+            error: {
+              type: 'mapper_parsing_exception',
+              reason:
+                "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'",
+              caused_by: {
+                type: 'illegal_state_exception',
+                reason: "Can't get text on a START_OBJECT at 1:3845",
+              },
+            },
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual({
+      ...responseBody,
+      items: [
+        responseBody.items[0],
+        responseBody.items[1],
+        {
+          create: {
+            _index: 'index1',
+            _id: '7',
+            status: 404,
+            error: {
+              type: 'mapper_parsing_exception',
+              reason:
+                "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.",
+              caused_by: {
+                type: 'illegal_state_exception',
+                reason: "Can't get text on a START_OBJECT at 1:3845",
+              },
+            },
+          },
+        },
+      ],
+    });
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual({
+      ...transportResponseBody,
+      body: {
+        ...transportResponseBody.body,
+        items: [
+          transportResponseBody.body.items[0],
+          transportResponseBody.body.items[1],
+          {
+            create: {
+              _index: 'index1',
+              _id: '7',
+              status: 404,
+              error: {
+                type: 'mapper_parsing_exception',
+                reason:
+                  "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.",
+                caused_by: {
+                  type: 'illegal_state_exception',
+                  reason: "Can't get text on a START_OBJECT at 1:3845",
+                },
+              },
+            },
+          },
+        ],
+      },
+    });
+  });
+});
+
+function wrapResponseBody(
+  body: estypes.BulkResponse,
+  statusCode: number = 200
+): TransportResult<estypes.BulkResponse, unknown> {
+  return {
+    body,
+    statusCode,
+    headers: {},
+    warnings: null,
+    // @ts-expect-error
+    meta: {},
+  };
+}

x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { cloneDeep } from 'lodash';
+import { TransportResult } from '@elastic/elasticsearch';
+import { get } from 'lodash';
+import { set } from '@kbn/safer-lodash-set';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+export const sanitizeBulkErrorResponse = (
+  response: TransportResult<estypes.BulkResponse, unknown> | estypes.BulkResponse
+): TransportResult<estypes.BulkResponse, unknown> | estypes.BulkResponse => {
+  const clonedResponse = cloneDeep(response);
+  const isTransportResponse = !!(response as TransportResult<estypes.BulkResponse, unknown>).body;
+
+  const responseToUse: estypes.BulkResponse = isTransportResponse
+    ? (clonedResponse as TransportResult<estypes.BulkResponse, unknown>).body
+    : (clonedResponse as estypes.BulkResponse);
+
+  if (responseToUse.errors) {
+    (responseToUse.items ?? []).forEach(
+      (item: Partial<Record<estypes.BulkOperationType, estypes.BulkResponseItem>>) => {
+        for (const [_, responseItem] of Object.entries(item)) {
+          const reason: string = get(responseItem, 'error.reason');
+          const redactIndex = reason ? reason.indexOf(`Preview of field's value:`) : -1;
+          if (redactIndex > 1) {
+            set(responseItem, 'error.reason', reason.substring(0, redactIndex - 1));
+          }
+        }
+      }
+    );
+  }
+
+  return clonedResponse;
+};

x-pack/plugins/alerting/server/index.ts

@@ -67,6 +67,7 @@ export {
   isValidAlertIndexName,
   InstallShutdownError,
 } from './alerts_service';
+export { sanitizeBulkErrorResponse } from './alerts_client';
 export { getDataStreamAdapter } from './alerts_service/lib/data_stream_adapter';
 
 export const plugin = async (initContext: PluginInitializerContext) => {