[8.12] [Security Solution] [Elastic AI Assistant] Include acknowledged alerts in the context sent to the LLM (Retrieval Augmented Generation (RAG) for Alerts) (#173121)

x-pack/packages/kbn-elastic-assistant/impl/alerts/settings/alerts_settings.tsx

@@ -109,6 +109,18 @@ const AlertsSettingsComponent = ({ knowledgeBase, setUpdatedKnowledgeBaseSetting
             <span>{i18n.LATEST_AND_RISKIEST_OPEN_ALERTS}</span>
           </EuiText>
         </EuiFlexItem>
+
+        <EuiFlexItem grow={false}>
+          <EuiText color="subdued" size="xs">
+            <span>{i18n.YOUR_ANONYMIZATION_SETTINGS}</span>
+          </EuiText>
+        </EuiFlexItem>
+
+        <EuiFlexItem grow={false}>
+          <EuiText color="subdued" size="xs">
+            <span>{i18n.SELECT_FEWER_ALERTS}</span>
+          </EuiText>
+        </EuiFlexItem>
       </EuiFlexGroup>
     </>
   );

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/translations.ts

@@ -23,9 +23,23 @@ export const ASK_QUESTIONS_ABOUT = i18n.translate(
 
 export const LATEST_AND_RISKIEST_OPEN_ALERTS = i18n.translate(
   'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.latestAndRiskiestOpenAlertsLabel',
+  {
+    defaultMessage: 'latest and riskiest open and acknowledged alerts in your environment.',
+  }
+);
+
+export const YOUR_ANONYMIZATION_SETTINGS = i18n.translate(
+  'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.yourAnonymizationSettingsLabel',
+  {
+    defaultMessage: 'Your Anonymization settings will be applied to the alerts.',
+  }
+);
+
+export const SELECT_FEWER_ALERTS = i18n.translate(
+  'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.selectFewerAlertsLabel',
   {
     defaultMessage:
-      'latest and riskiest open alerts in your environment. Your Anonymization settings will be applied to the alerts',
+      "Select fewer alerts if the model's maximum context length is frequently exceeded.",
   }
 );
 

x-pack/plugins/elastic_assistant/server/lib/langchain/tools/index.ts

@@ -12,7 +12,7 @@ import { Tool } from 'langchain/tools';
 
 import { getAlertCountsTool } from './alert_counts/get_alert_counts_tool';
 import { getEsqlLanguageKnowledgeBaseTool } from './esql_language_knowledge_base/get_esql_language_knowledge_base_tool';
-import { getOpenAlertsTool } from './open_alerts/get_open_alerts_tool';
+import { getOpenAndAcknowledgedAlertsTool } from './open_and_acknowledged_alerts/get_open_and_acknowledged_alerts_tool';
 import type { RequestBody } from '../types';
 
 export interface GetApplicableTools {
@@ -50,7 +50,7 @@ export const getApplicableTools = ({
       replacements,
       request,
     }) ?? [],
-    getOpenAlertsTool({
+    getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow,
       allowReplacement,

x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_alerts/get_open_alerts_query.test.ts → x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_and_acknowledged_alerts/get_open_and_acknowledged_alerts_query.test.ts

@@ -5,15 +5,15 @@
  * 2.0.
  */
 
-import { getOpenAlertsQuery } from './get_open_alerts_query';
+import { getOpenAndAcknowledgedAlertsQuery } from './get_open_and_acknowledged_alerts_query';
 
-describe('getOpenAlertsQuery', () => {
+describe('getOpenAndAcknowledgedAlertsQuery', () => {
   it('returns the expected query', () => {
     const alertsIndexPattern = 'alerts-*';
     const allow = ['field1', 'field2'];
     const size = 10;
 
-    const query = getOpenAlertsQuery({ alertsIndexPattern, allow, size });
+    const query = getOpenAndAcknowledgedAlertsQuery({ alertsIndexPattern, allow, size });
 
     expect(query).toEqual({
       allow_no_indices: true,
@@ -30,8 +30,20 @@ describe('getOpenAlertsQuery', () => {
                   must: [],
                   filter: [
                     {
-                      match_phrase: {
-                        'kibana.alert.workflow_status': 'open',
+                      bool: {
+                        should: [
+                          {
+                            match_phrase: {
+                              'kibana.alert.workflow_status': 'open',
+                            },
+                          },
+                          {
+                            match_phrase: {
+                              'kibana.alert.workflow_status': 'acknowledged',
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
                       },
                     },
                     {

x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_alerts/get_open_alerts_query.ts → x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_and_acknowledged_alerts/get_open_and_acknowledged_alerts_query.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-export const getOpenAlertsQuery = ({
+export const getOpenAndAcknowledgedAlertsQuery = ({
   alertsIndexPattern,
   allow,
   size,
@@ -28,8 +28,20 @@ export const getOpenAlertsQuery = ({
               must: [],
               filter: [
                 {
-                  match_phrase: {
-                    'kibana.alert.workflow_status': 'open',
+                  bool: {
+                    should: [
+                      {
+                        match_phrase: {
+                          'kibana.alert.workflow_status': 'open',
+                        },
+                      },
+                      {
+                        match_phrase: {
+                          'kibana.alert.workflow_status': 'acknowledged',
+                        },
+                      },
+                    ],
+                    minimum_should_match: 1,
                   },
                 },
                 {

x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_alerts/get_open_alerts_tool.test.ts → x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_and_acknowledged_alerts/get_open_and_acknowledged_alerts_tool.test.ts

@@ -10,12 +10,12 @@ import type { KibanaRequest } from '@kbn/core-http-server';
 import { DynamicTool } from 'langchain/tools';
 import { omit } from 'lodash/fp';
 
-import { getOpenAlertsTool } from './get_open_alerts_tool';
+import { getOpenAndAcknowledgedAlertsTool } from './get_open_and_acknowledged_alerts_tool';
 import { mockAlertsFieldsApi } from '../../../../__mocks__/alerts';
 import type { RequestBody } from '../../types';
 import { MAX_SIZE } from './helpers';
 
-describe('getOpenAlertsTool', () => {
+describe('getOpenAndAcknowledgedAlertsTool', () => {
   const alertsIndexPattern = 'alerts-index';
   const esClient = {
     search: jest.fn().mockResolvedValue(mockAlertsFieldsApi),
@@ -37,7 +37,7 @@ describe('getOpenAlertsTool', () => {
   });
 
   it('returns a `DynamicTool` with a `func` that calls `esClient.search()` with the expected query', async () => {
-    const tool: DynamicTool = getOpenAlertsTool({
+    const tool: DynamicTool = getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow: request.body.allow,
       allowReplacement: request.body.allowReplacement,
@@ -75,8 +75,20 @@ describe('getOpenAlertsTool', () => {
                 bool: {
                   filter: [
                     {
-                      match_phrase: {
-                        'kibana.alert.workflow_status': 'open',
+                      bool: {
+                        should: [
+                          {
+                            match_phrase: {
+                              'kibana.alert.workflow_status': 'open',
+                            },
+                          },
+                          {
+                            match_phrase: {
+                              'kibana.alert.workflow_status': 'acknowledged',
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
                       },
                     },
                     {
@@ -130,7 +142,7 @@ describe('getOpenAlertsTool', () => {
       RequestBody
     >;
 
-    const tool = getOpenAlertsTool({
+    const tool = getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow: requestWithMissingParams.body.allow,
       allowReplacement: requestWithMissingParams.body.allowReplacement,
@@ -145,7 +157,7 @@ describe('getOpenAlertsTool', () => {
   });
 
   it('returns null when alertsIndexPattern is undefined', () => {
-    const tool = getOpenAlertsTool({
+    const tool = getOpenAndAcknowledgedAlertsTool({
       // alertsIndexPattern is undefined
       allow: request.body.allow,
       allowReplacement: request.body.allowReplacement,
@@ -160,7 +172,7 @@ describe('getOpenAlertsTool', () => {
   });
 
   it('returns null when size is undefined', () => {
-    const tool = getOpenAlertsTool({
+    const tool = getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow: request.body.allow,
       allowReplacement: request.body.allowReplacement,
@@ -175,7 +187,7 @@ describe('getOpenAlertsTool', () => {
   });
 
   it('returns null when size out of range', () => {
-    const tool = getOpenAlertsTool({
+    const tool = getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow: request.body.allow,
       allowReplacement: request.body.allowReplacement,
@@ -190,7 +202,7 @@ describe('getOpenAlertsTool', () => {
   });
 
   it('returns a tool instance with the expected tags', () => {
-    const tool = getOpenAlertsTool({
+    const tool = getOpenAndAcknowledgedAlertsTool({
       alertsIndexPattern,
       allow: request.body.allow,
       allowReplacement: request.body.allowReplacement,

x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_alerts/get_open_alerts_tool.ts → x-pack/plugins/elastic_assistant/server/lib/langchain/tools/open_and_acknowledged_alerts/get_open_and_acknowledged_alerts_tool.ts

@@ -13,17 +13,17 @@ import { DynamicTool, Tool } from 'langchain/tools';
 import { requestHasRequiredAnonymizationParams } from '../../helpers';
 import { RequestBody } from '../../types';
 
-import { getOpenAlertsQuery } from './get_open_alerts_query';
+import { getOpenAndAcknowledgedAlertsQuery } from './get_open_and_acknowledged_alerts_query';
 import { getRawDataOrDefault, sizeIsOutOfRange } from './helpers';
 
 export const OPEN_ALERTS_TOOL_DESCRIPTION =
   'Call this for knowledge about the latest n open alerts (sorted by `kibana.alert.risk_score`) in the environment, or when answering questions about open alerts';
 
 /**
- * Returns a tool for querying open alerts, or null if the request
- * doesn't have all the required parameters.
+ * Returns a tool for querying open and acknowledged alerts, or null if the
+ * request doesn't have all the required parameters.
  */
-export const getOpenAlertsTool = ({
+export const getOpenAndAcknowledgedAlertsTool = ({
   alertsIndexPattern,
   allow,
   allowReplacement,
@@ -55,7 +55,7 @@ export const getOpenAlertsTool = ({
     name: 'open-alerts',
     description: OPEN_ALERTS_TOOL_DESCRIPTION,
     func: async () => {
-      const query = getOpenAlertsQuery({
+      const query = getOpenAndAcknowledgedAlertsQuery({
         alertsIndexPattern,
         allow: allow ?? [],
         size,

x-pack/plugins/security_solution/public/assistant/content/anonymization/index.ts

@@ -7,6 +7,7 @@
 
 /** By default, these fields are allowed to be sent to the assistant */
 export const DEFAULT_ALLOW = [
+  '_id',
   '@timestamp',
   'cloud.availability_zone',
   'cloud.provider',
@@ -28,6 +29,7 @@ export const DEFAULT_ALLOW = [
   'host.risk.calculated_level',
   'host.risk.calculated_score_norm',
   'kibana.alert.last_detected',
+  'kibana.alert.risk_score',
   'kibana.alert.rule.description',
   'kibana.alert.rule.name',
   'kibana.alert.rule.references',
@@ -42,6 +44,7 @@ export const DEFAULT_ALLOW = [
   'kibana.alert.rule.threat.technique.subtechnique.name',
   'kibana.alert.rule.threat.technique.subtechnique.reference',
   'kibana.alert.severity',
+  'kibana.alert.workflow_status',
   'process.args',
   'process.command_line',
   'process.executable',
@@ -73,6 +76,7 @@ export const DEFAULT_ALLOW = [
 
 /** By default, these fields will be anonymized */
 export const DEFAULT_ALLOW_REPLACEMENT = [
+  '_id', // the document's _id is replaced with an anonymized value
   'cloud.availability_zone',
   'cloud.provider',
   'cloud.region',