From fd9e33ccea41052fa2d0006a317ca173a184fcc8 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 5 Dec 2023 15:33:33 -0800 Subject: [PATCH 001/141] [Security solution] AI assistant persistent storage --- x-pack/plugins/elastic_assistant/kibana.jsonc | 3 +- .../assistant_data_writer.ts | 88 ++ .../server/ai_assistant_data_client/index.ts | 61 ++ .../create_resource_installation_helper.ts | 139 +++ .../component_template_from_field_map.test.ts | 79 ++ .../component_template_from_field_map.ts | 43 + .../field_maps/mapping_from_field_map.test.ts | 382 ++++++++ .../field_maps/mapping_from_field_map.ts | 53 ++ .../server/ai_assistant_service/index.test.ts | 844 ++++++++++++++++++ .../server/ai_assistant_service/index.ts | 294 ++++++ .../lib/conversation_configuration_type.ts | 182 ++++ .../lib/create_concrete_write_index.test.ts | 718 +++++++++++++++ .../lib/create_concrete_write_index.ts | 164 ++++ .../lib/create_datastream.mock.ts | 17 + .../lib/create_datastream.ts | 89 ++ .../lib/get_configuration_index.test.ts | 59 ++ .../lib/get_configuration_index.ts | 44 + .../ai_assistant_service/lib/indices.ts | 15 + .../lib/retry_transient_es_errors.ts | 58 ++ .../elastic_assistant/server/plugin.ts | 74 +- .../server/routes/request_context_factory.ts | 88 ++ .../ai_assistant_default_prompts.ts | 61 ++ .../saved_object/ai_assistant_so_client.ts | 267 ++++++ .../assistant_prompts_so_schema.ts | 79 ++ ...tic_assistant_anonimization_fields_type.ts | 45 + .../elastic_assistant_prompts_type.ts | 131 +++ .../server/saved_object/index.ts | 9 + .../server/schemas/conversation_apis.yml | 439 +++++++++ .../plugins/elastic_assistant/server/types.ts | 72 ++ 29 files changed, 4569 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml diff --git a/x-pack/plugins/elastic_assistant/kibana.jsonc b/x-pack/plugins/elastic_assistant/kibana.jsonc index 395da30fcf359..cb89de90fe649 100644 --- a/x-pack/plugins/elastic_assistant/kibana.jsonc +++ b/x-pack/plugins/elastic_assistant/kibana.jsonc @@ -10,7 +10,8 @@ "requiredPlugins": [ "actions", "data", - "ml" + "ml", + "taskManager" ] } } diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts new file mode 100644 index 0000000000000..dbdbb1b3b2ce0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; + +interface WriterBulkResponse { + errors: string[]; + docs_written: number; + took: number; +} + +interface BulkParams { + conversations?: string[]; +} + +export interface AssistantDataWriter { + bulk: (params: BulkParams) => Promise; +} + +interface AssistantDataWriterOptions { + esClient: ElasticsearchClient; + index: string; + namespace: string; + logger: Logger; +} + +export class AssistantDataWriter implements AssistantDataWriter { + constructor(private readonly options: AssistantDataWriterOptions) {} + + public bulk = async (params: BulkParams) => { + try { + if (!params.conversations?.length) { + return { errors: [], docs_written: 0, took: 0 }; + } + + const { errors, items, took } = await this.options.esClient.bulk({ + operations: this.buildBulkOperations(params), + }); + + return { + errors: errors + ? items + .map((item) => item.create?.error?.reason) + .filter((error): error is string => !!error) + : [], + docs_written: items.filter( + (item) => item.create?.status === 201 || item.create?.status === 200 + ).length, + took, + }; + } catch (e) { + this.options.logger.error(`Error writing risk scores: ${e.message}`); + return { + errors: [`${e.message}`], + docs_written: 0, + took: 0, + }; + } + }; + + private buildBulkOperations = (params: BulkParams): BulkOperationContainer[] => { + const conversationBody = + params.conversations?.flatMap((conversation) => [ + { create: { _index: this.options.index } }, + // this.scoreToEcs(score, 'host'), + ]) ?? []; + + return conversationBody as BulkOperationContainer[]; + }; + + /* private scoreToEcs = (score: unknown, identifierType: string): unknown => { + const { '@timestamp': _, ...rest } = score; + return { + '@timestamp': score['@timestamp'], + [identifierType]: { + name: score.id_value, + risk: { + ...rest, + }, + }, + }; + };*/ +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts new file mode 100644 index 0000000000000..c1fcb2e5ac02e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { chunk, flatMap, get, keys } from 'lodash'; +import { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { IIndexPatternString } from '../types'; +import { AssistantDataWriter } from './assistant_data_writer'; +import { getIndexTemplateAndPattern } from '../ai_assistant_service/lib/conversation_configuration_type'; + +// Term queries can take up to 10,000 terms +const CHUNK_SIZE = 10000; + +export interface AIAssistantDataClientParams { + elasticsearchClientPromise: Promise; + kibanaVersion: string; + namespace: string; + logger: Logger; + indexPatternsResorceName: string; +} + +export class AIAssistantDataClient { + private writerCache: Map = new Map(); + + private indexTemplateAndPattern: IIndexPatternString; + + constructor(private readonly options: AIAssistantDataClientParams) { + this.indexTemplateAndPattern = getIndexTemplateAndPattern( + this.options.indexPatternsResorceName, + this.options.namespace ?? DEFAULT_NAMESPACE_STRING + ); + } + + public async getWriter({ namespace }: { namespace: string }): Promise { + if (this.writerCache.get(namespace)) { + return this.writerCache.get(namespace) as AssistantDataWriter; + } + const indexPatterns = this.indexTemplateAndPattern; + await this.initializeWriter(namespace, indexPatterns.alias); + return this.writerCache.get(namespace) as AssistantDataWriter; + } + + private async initializeWriter(namespace: string, index: string): Promise { + const esClient = await this.options.elasticsearchClientPromise; + const writer = new AssistantDataWriter({ + esClient, + namespace, + index, + logger: this.options.logger, + }); + + this.writerCache.set(namespace, writer); + return writer; + } +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts new file mode 100644 index 0000000000000..6d238514b0ba9 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { Logger } from '@kbn/core/server'; + +export interface InitializationPromise { + result: boolean; + error?: string; +} + +// get multiples of 2 min +const DEFAULT_RETRY_BACKOFF_PER_ATTEMPT = 2 * 60 * 1000; +interface Retry { + time: string; // last time retry was requested + attempts: number; // number of retry attemps +} +export interface ResourceInstallationHelper { + add: (namespace?: string, timeoutMs?: number) => void; + retry: ( + namespace?: string, + initPromise?: Promise, + timeoutMs?: number + ) => void; + getInitializedResources: (namespace: string) => Promise; +} + +/** + * Helper function that queues up resources to initialize until we are + * ready to begin initialization. Once we're ready, we start taking from + * the queue and kicking off initialization. + * + * If a resource is added after we begin initialization, we push it onto + * the queue and the running loop will handle it + * + * If a resource is added to the queue when the processing loop is not + * running, kick off the processing loop + */ +export function createResourceInstallationHelper( + logger: Logger, + commonResourcesInitPromise: Promise, + installFn: (namespace: string, timeoutMs?: number) => Promise +): ResourceInstallationHelper { + let commonInitPromise: Promise = commonResourcesInitPromise; + const initializedResources: Map> = new Map(); + const lastRetry: Map = new Map(); + + const waitUntilResourcesInstalled = async ( + namespace: string = DEFAULT_NAMESPACE_STRING, + timeoutMs?: number + ): Promise => { + try { + const { result: commonInitResult, error: commonInitError } = await commonInitPromise; + if (commonInitResult) { + await installFn(namespace, timeoutMs); + return successResult(); + } else { + logger.warn( + `Common resources were not initialized, cannot initialize resources for ${namespace}` + ); + return errorResult(commonInitError); + } + } catch (err) { + logger.error(`Error initializing resources ${namespace} - ${err.message}`); + return errorResult(err.message); + } + }; + + return { + add: (namespace: string = DEFAULT_NAMESPACE_STRING, timeoutMs?: number) => { + initializedResources.set( + `${namespace}`, + + // Return a promise than can be checked when needed + waitUntilResourcesInstalled(namespace, timeoutMs) + ); + }, + retry: ( + namespace: string = DEFAULT_NAMESPACE_STRING, + initPromise?: Promise, + timeoutMs?: number + ) => { + const key = `${namespace}`; + // Use the new common initialization promise if specified + if (initPromise) { + commonInitPromise = initPromise; + } + + // Check the last retry time to see if we want to throttle this attempt + const retryInfo = lastRetry.get(key); + const shouldRetry = retryInfo ? getShouldRetry(retryInfo) : true; + + if (shouldRetry) { + logger.info(`Retrying resource initialization for "${namespace}"`); + // Update the last retry information + lastRetry.set(key, { + time: new Date().toISOString(), + attempts: (retryInfo?.attempts ?? 0) + 1, + }); + + initializedResources.set( + key, + // Return a promise than can be checked when needed + waitUntilResourcesInstalled(namespace, timeoutMs) + ); + } + }, + getInitializedResources: async (namespace: string): Promise => { + const key = `${namespace}`; + return ( + initializedResources.has(key) + ? initializedResources.get(key) + : errorResult(`Unrecognized resources key ${key}`) + ) as InitializationPromise; + }, + }; +} + +export const successResult = () => ({ result: true }); +export const errorResult = (error?: string) => ({ result: false, error }); + +export const getShouldRetry = ({ time, attempts }: Retry) => { + const now = new Date().valueOf(); + const nextRetryDate = new Date(time).valueOf() + calculateDelay(attempts); + return now > nextRetryDate; +}; + +export const calculateDelay = (attempts: number) => { + if (attempts === 1) { + return 30 * 1000; // 30s + } else { + // 2, 4, 6, 8, etc minutes + return DEFAULT_RETRY_BACKOFF_PER_ATTEMPT * Math.pow(2, attempts - 2); + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts new file mode 100644 index 0000000000000..17bde15bd01a5 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getComponentTemplateFromFieldMap } from './component_template_from_field_map'; +import { testFieldMap, expectedTestMapping } from './mapping_from_field_map.test'; + +describe('getComponentTemplateFromFieldMap', () => { + it('correctly creates component template from field map', () => { + expect( + getComponentTemplateFromFieldMap({ name: 'test-mappings', fieldMap: testFieldMap }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: {}, + mappings: { + dynamic: 'strict', + ...expectedTestMapping, + }, + }, + }); + }); + + it('correctly creates component template with settings when includeSettings = true', () => { + expect( + getComponentTemplateFromFieldMap({ + name: 'test-mappings', + fieldMap: testFieldMap, + includeSettings: true, + }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: 'strict', + ...expectedTestMapping, + }, + }, + }); + }); + + it('correctly creates component template with dynamic setting when defined', () => { + expect( + getComponentTemplateFromFieldMap({ + name: 'test-mappings', + fieldMap: testFieldMap, + includeSettings: true, + dynamic: false, + }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: false, + ...expectedTestMapping, + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts new file mode 100644 index 0000000000000..9d5c651e92cda --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { type FieldMap } from '@kbn/alerts-as-data-utils'; +import { mappingFromFieldMap } from './mapping_from_field_map'; + +export interface GetComponentTemplateFromFieldMapOpts { + name: string; + fieldMap: FieldMap; + includeSettings?: boolean; + dynamic?: 'strict' | false; +} +export const getComponentTemplateFromFieldMap = ({ + name, + fieldMap, + dynamic, + includeSettings, +}: GetComponentTemplateFromFieldMapOpts): ClusterPutComponentTemplateRequest => { + return { + name, + _meta: { + managed: true, + }, + template: { + settings: { + ...(includeSettings + ? { + number_of_shards: 1, + 'index.mapping.total_fields.limit': + Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, + } + : {}), + }, + + mappings: mappingFromFieldMap(fieldMap, dynamic ?? 'strict'), + }, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts new file mode 100644 index 0000000000000..e58b795863e48 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts @@ -0,0 +1,382 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; +import { mappingFromFieldMap } from './mapping_from_field_map'; + +export const testFieldMap: FieldMap = { + date_field: { + type: 'date', + array: false, + required: true, + }, + keyword_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + long_field: { + type: 'long', + array: false, + required: false, + }, + multifield_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + multi_fields: [ + { + flat_name: 'multifield_field.text', + name: 'text', + type: 'match_only_text', + }, + ], + }, + geopoint_field: { + type: 'geo_point', + array: false, + required: false, + }, + ip_field: { + type: 'ip', + array: false, + required: false, + }, + array_field: { + type: 'keyword', + array: true, + required: false, + ignore_above: 1024, + }, + nested_array_field: { + type: 'nested', + array: false, + required: false, + }, + 'nested_array_field.field1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'nested_array_field.field2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + scaled_float_field: { + type: 'scaled_float', + array: false, + required: false, + scaling_factor: 1000, + }, + constant_keyword_field: { + type: 'constant_keyword', + array: false, + required: false, + }, + 'parent_field.child1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'parent_field.child2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + unmapped_object: { + type: 'object', + required: false, + enabled: false, + }, + formatted_field: { + type: 'date_range', + required: false, + format: 'epoch_millis||strict_date_optional_time', + }, +}; +export const expectedTestMapping = { + properties: { + array_field: { + ignore_above: 1024, + type: 'keyword', + }, + constant_keyword_field: { + type: 'constant_keyword', + }, + date_field: { + type: 'date', + }, + multifield_field: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + geopoint_field: { + type: 'geo_point', + }, + ip_field: { + type: 'ip', + }, + keyword_field: { + ignore_above: 1024, + type: 'keyword', + }, + long_field: { + type: 'long', + }, + nested_array_field: { + properties: { + field1: { + ignore_above: 1024, + type: 'keyword', + }, + field2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + parent_field: { + properties: { + child1: { + ignore_above: 1024, + type: 'keyword', + }, + child2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + scaled_float_field: { + scaling_factor: 1000, + type: 'scaled_float', + }, + unmapped_object: { + enabled: false, + type: 'object', + }, + formatted_field: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + }, +}; + +describe('mappingFromFieldMap', () => { + it('correctly creates mapping from field map', () => { + expect(mappingFromFieldMap(testFieldMap)).toEqual({ + dynamic: 'strict', + ...expectedTestMapping, + }); + expect(mappingFromFieldMap(alertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + '@timestamp': { + ignore_malformed: false, + type: 'date', + }, + event: { + properties: { + action: { + type: 'keyword', + }, + kind: { + type: 'keyword', + }, + }, + }, + kibana: { + properties: { + alert: { + properties: { + action_group: { + type: 'keyword', + }, + case_ids: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + end: { + type: 'date', + }, + flapping: { + type: 'boolean', + }, + flapping_history: { + type: 'boolean', + }, + maintenance_window_ids: { + type: 'keyword', + }, + instance: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + last_detected: { + type: 'date', + }, + reason: { + type: 'keyword', + }, + rule: { + properties: { + category: { + type: 'keyword', + }, + consumer: { + type: 'keyword', + }, + execution: { + properties: { + uuid: { + type: 'keyword', + }, + }, + }, + name: { + type: 'keyword', + }, + parameters: { + type: 'flattened', + ignore_above: 4096, + }, + producer: { + type: 'keyword', + }, + revision: { + type: 'long', + }, + rule_type_id: { + type: 'keyword', + }, + tags: { + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + }, + }, + start: { + type: 'date', + }, + status: { + type: 'keyword', + }, + time_range: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + url: { + ignore_above: 2048, + index: false, + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + workflow_status: { + type: 'keyword', + }, + workflow_tags: { + type: 'keyword', + }, + }, + }, + space_ids: { + type: 'keyword', + }, + version: { + type: 'version', + }, + }, + }, + tags: { + type: 'keyword', + }, + }, + }); + expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + kibana: { + properties: { + alert: { + properties: { + risk_score: { type: 'float' }, + rule: { + properties: { + author: { type: 'keyword' }, + created_at: { type: 'date' }, + created_by: { type: 'keyword' }, + description: { type: 'keyword' }, + enabled: { type: 'keyword' }, + from: { type: 'keyword' }, + interval: { type: 'keyword' }, + license: { type: 'keyword' }, + note: { type: 'keyword' }, + references: { type: 'keyword' }, + rule_id: { type: 'keyword' }, + rule_name_override: { type: 'keyword' }, + to: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + updated_by: { type: 'keyword' }, + version: { type: 'keyword' }, + }, + }, + severity: { type: 'keyword' }, + suppression: { + properties: { + docs_count: { type: 'long' }, + end: { type: 'date' }, + terms: { + properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, + }, + start: { type: 'date' }, + }, + }, + system_status: { type: 'keyword' }, + workflow_reason: { type: 'keyword' }, + workflow_user: { type: 'keyword' }, + }, + }, + }, + }, + ecs: { properties: { version: { type: 'keyword' } } }, + }, + }); + }); + + it('uses dynamic setting if specified', () => { + expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ + dynamic: true, + ...expectedTestMapping, + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts new file mode 100644 index 0000000000000..1d5121883df69 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { set } from '@kbn/safer-lodash-set'; +import type { FieldMap, MultiField } from '@kbn/alerts-as-data-utils'; + +export function mappingFromFieldMap( + fieldMap: FieldMap, + dynamic: 'strict' | boolean = 'strict' +): MappingTypeMapping { + const mappings = { + dynamic, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key: string) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { name, required, array, multi_fields, ...rest } = field; + const mapped = multi_fields + ? { + ...rest, + // eslint-disable-next-line @typescript-eslint/naming-convention + fields: multi_fields.reduce((acc, multi_field: MultiField) => { + acc[multi_field.name] = { + type: multi_field.type, + }; + return acc; + }, {} as Record), + } + : rest; + + set(mappings.properties, field.name.split('.').join('.properties.'), mapped); + + if (name === '@timestamp') { + set(mappings.properties, `${name}.ignore_malformed`, false); + } + }); + + return mappings; +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts new file mode 100644 index 0000000000000..acbb1b1e60029 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -0,0 +1,844 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createOrUpdateComponentTemplate, + createOrUpdateIndexTemplate, +} from '@kbn/alerting-plugin/server'; +import { + loggingSystemMock, + elasticsearchServiceMock, + savedObjectsClientMock, +} from '@kbn/core/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import type { SavedObject } from '@kbn/core/server'; + +const getSavedObjectConfiguration = (attributes = {}) => ({ + page: 1, + per_page: 20, + total: 1, + saved_objects: [ + { + type: 'risk-engine-configuration', + id: 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', + namespaces: ['default'], + attributes: { + enabled: false, + ...attributes, + }, + references: [], + managed: false, + updated_at: '2023-07-28T09:52:28.768Z', + created_at: '2023-07-28T09:12:26.083Z', + version: 'WzE4MzIsMV0=', + coreMigrationVersion: '8.8.0', + score: 0, + }, + ], +}); + +const transformsMock = { + count: 1, + transforms: [ + { + id: 'ml_hostriskscore_pivot_transform_default', + dest: { index: '' }, + source: { index: '' }, + }, + ], +}; + +jest.mock('@kbn/alerting-plugin/server', () => ({ + createOrUpdateComponentTemplate: jest.fn(), + createOrUpdateIndexTemplate: jest.fn(), +})); + +jest.mock('./utils/create_datastream', () => ({ + createDataStream: jest.fn(), +})); + +jest.mock('../../risk_score/transform/helpers/transforms', () => ({ + createAndStartTransform: jest.fn(), +})); + +jest.mock('./utils/create_index', () => ({ + createIndex: jest.fn(), +})); + +jest.spyOn(transforms, 'createTransform').mockResolvedValue(Promise.resolve()); +jest.spyOn(transforms, 'startTransform').mockResolvedValue(Promise.resolve()); + +describe('RiskEngineDataClient', () => { + for (const useDataStreamForAlerts of [false, true]) { + const label = useDataStreamForAlerts ? 'data streams' : 'aliases'; + + describe(`using ${label} for alert indices`, () => { + let riskEngineDataClient: RiskEngineDataClient; + let mockSavedObjectClient: ReturnType; + let logger: ReturnType; + const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; + const totalFieldsLimit = 1000; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + mockSavedObjectClient = savedObjectsClientMock.create(); + const options = { + logger, + kibanaVersion: '8.9.0', + esClient, + soClient: mockSavedObjectClient, + namespace: 'default', + }; + riskEngineDataClient = new RiskEngineDataClient(options); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('getWriter', () => { + it('should return a writer object', async () => { + const writer = await riskEngineDataClient.getWriter({ namespace: 'default' }); + expect(writer).toBeDefined(); + expect(typeof writer?.bulk).toBe('function'); + }); + + it('should cache and return the same writer for the same namespace', async () => { + const writer1 = await riskEngineDataClient.getWriter({ namespace: 'default' }); + const writer2 = await riskEngineDataClient.getWriter({ namespace: 'default' }); + const writer3 = await riskEngineDataClient.getWriter({ namespace: 'space-1' }); + + expect(writer1).toEqual(writer2); + expect(writer2).not.toEqual(writer3); + }); + }); + + describe('initializeResources success', () => { + it('should initialize risk engine resources', async () => { + await riskEngineDataClient.initializeResources({ namespace: 'default' }); + + expect(createOrUpdateComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + logger, + esClient, + template: expect.objectContaining({ + name: '.risk-score-mappings', + _meta: { + managed: true, + }, + }), + totalFieldsLimit: 1000, + }) + ); + expect((createOrUpdateComponentTemplate as jest.Mock).mock.lastCall[0].template.template) + .toMatchInlineSnapshot(` + Object { + "mappings": Object { + "dynamic": "strict", + "properties": Object { + "@timestamp": Object { + "ignore_malformed": false, + "type": "date", + }, + "host": Object { + "properties": Object { + "name": Object { + "type": "keyword", + }, + "risk": Object { + "properties": Object { + "calculated_level": Object { + "type": "keyword", + }, + "calculated_score": Object { + "type": "float", + }, + "calculated_score_norm": Object { + "type": "float", + }, + "category_1_count": Object { + "type": "long", + }, + "category_1_score": Object { + "type": "float", + }, + "id_field": Object { + "type": "keyword", + }, + "id_value": Object { + "type": "keyword", + }, + "inputs": Object { + "properties": Object { + "category": Object { + "type": "keyword", + }, + "description": Object { + "type": "keyword", + }, + "id": Object { + "type": "keyword", + }, + "index": Object { + "type": "keyword", + }, + "risk_score": Object { + "type": "float", + }, + "timestamp": Object { + "type": "date", + }, + }, + "type": "object", + }, + "notes": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + }, + }, + "user": Object { + "properties": Object { + "name": Object { + "type": "keyword", + }, + "risk": Object { + "properties": Object { + "calculated_level": Object { + "type": "keyword", + }, + "calculated_score": Object { + "type": "float", + }, + "calculated_score_norm": Object { + "type": "float", + }, + "category_1_count": Object { + "type": "long", + }, + "category_1_score": Object { + "type": "float", + }, + "id_field": Object { + "type": "keyword", + }, + "id_value": Object { + "type": "keyword", + }, + "inputs": Object { + "properties": Object { + "category": Object { + "type": "keyword", + }, + "description": Object { + "type": "keyword", + }, + "id": Object { + "type": "keyword", + }, + "index": Object { + "type": "keyword", + }, + "risk_score": Object { + "type": "float", + }, + "timestamp": Object { + "type": "date", + }, + }, + "type": "object", + }, + "notes": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + }, + }, + }, + }, + "settings": Object {}, + } + `); + + expect(createOrUpdateIndexTemplate).toHaveBeenCalledWith({ + logger, + esClient, + template: { + name: '.risk-score.risk-score-default-index-template', + body: { + data_stream: { hidden: true }, + index_patterns: ['risk-score.risk-score-default'], + composed_of: ['.risk-score-mappings'], + template: { + lifecycle: {}, + settings: { + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: { + kibana: { + version: '8.9.0', + }, + managed: true, + namespace: 'default', + }, + }, + }, + _meta: { + kibana: { + version: '8.9.0', + }, + managed: true, + namespace: 'default', + }, + }, + }, + }); + + expect(createDataStream).toHaveBeenCalledWith({ + logger, + esClient, + totalFieldsLimit, + indexPatterns: { + template: `.risk-score.risk-score-default-index-template`, + alias: `risk-score.risk-score-default`, + }, + }); + + expect(createIndex).toHaveBeenCalledWith({ + logger, + esClient, + options: { + index: `risk-score.risk-score-latest-default`, + mappings: { + dynamic: 'strict', + properties: { + '@timestamp': { + ignore_malformed: false, + type: 'date', + }, + host: { + properties: { + name: { + type: 'keyword', + }, + risk: { + properties: { + calculated_level: { + type: 'keyword', + }, + calculated_score: { + type: 'float', + }, + calculated_score_norm: { + type: 'float', + }, + category_1_count: { + type: 'long', + }, + category_1_score: { + type: 'float', + }, + id_field: { + type: 'keyword', + }, + id_value: { + type: 'keyword', + }, + inputs: { + properties: { + category: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + risk_score: { + type: 'float', + }, + timestamp: { + type: 'date', + }, + }, + type: 'object', + }, + notes: { + type: 'keyword', + }, + }, + type: 'object', + }, + }, + }, + user: { + properties: { + name: { + type: 'keyword', + }, + risk: { + properties: { + calculated_level: { + type: 'keyword', + }, + calculated_score: { + type: 'float', + }, + calculated_score_norm: { + type: 'float', + }, + category_1_count: { + type: 'long', + }, + category_1_score: { + type: 'float', + }, + id_field: { + type: 'keyword', + }, + id_value: { + type: 'keyword', + }, + inputs: { + properties: { + category: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + risk_score: { + type: 'float', + }, + timestamp: { + type: 'date', + }, + }, + type: 'object', + }, + notes: { + type: 'keyword', + }, + }, + type: 'object', + }, + }, + }, + }, + }, + }, + }); + + expect(transforms.createTransform).toHaveBeenCalledWith({ + logger, + esClient, + transform: { + dest: { + index: 'risk-score.risk-score-latest-default', + }, + frequency: '1h', + latest: { + sort: '@timestamp', + unique_key: ['host.name', 'user.name'], + }, + source: { + index: ['risk-score.risk-score-default'], + }, + sync: { + time: { + delay: '2s', + field: '@timestamp', + }, + }, + transform_id: 'risk_score_latest_transform_default', + }, + }); + }); + }); + + describe('initializeResources error', () => { + it('should handle errors during initialization', async () => { + const error = new Error('There error'); + (createOrUpdateIndexTemplate as jest.Mock).mockRejectedValueOnce(error); + + try { + await riskEngineDataClient.initializeResources({ namespace: 'default' }); + } catch (e) { + expect(logger.error).toHaveBeenCalledWith( + `Error initializing risk engine resources: ${error.message}` + ); + } + }); + }); + + describe('getStatus', () => { + it('should return initial status', async () => { + const status = await riskEngineDataClient.getStatus({ + namespace: 'default', + }); + expect(status).toEqual({ + isMaxAmountOfRiskEnginesReached: false, + riskEngineStatus: 'NOT_INSTALLED', + legacyRiskEngineStatus: 'NOT_INSTALLED', + }); + }); + + describe('saved object exists and transforms not', () => { + beforeEach(() => { + mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); + }); + + it('should return status with enabled true', async () => { + mockSavedObjectClient.find.mockResolvedValue( + getSavedObjectConfiguration({ + enabled: true, + }) + ); + + const status = await riskEngineDataClient.getStatus({ + namespace: 'default', + }); + expect(status).toEqual({ + isMaxAmountOfRiskEnginesReached: true, + riskEngineStatus: 'ENABLED', + legacyRiskEngineStatus: 'NOT_INSTALLED', + }); + }); + + it('should return status with enabled false', async () => { + mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); + + const status = await riskEngineDataClient.getStatus({ + namespace: 'default', + }); + expect(status).toEqual({ + isMaxAmountOfRiskEnginesReached: false, + riskEngineStatus: 'DISABLED', + legacyRiskEngineStatus: 'NOT_INSTALLED', + }); + }); + }); + + describe('legacy transforms', () => { + it('should fetch transforms', async () => { + await riskEngineDataClient.getStatus({ + namespace: 'default', + }); + + expect(esClient.transform.getTransform).toHaveBeenCalledTimes(4); + expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(1, { + transform_id: 'ml_hostriskscore_pivot_transform_default', + }); + expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(2, { + transform_id: 'ml_hostriskscore_latest_transform_default', + }); + expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(3, { + transform_id: 'ml_userriskscore_pivot_transform_default', + }); + expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(4, { + transform_id: 'ml_userriskscore_latest_transform_default', + }); + }); + + it('should return that legacy transform enabled if at least on transform exist', async () => { + esClient.transform.getTransform.mockResolvedValueOnce(transformsMock); + + const status = await riskEngineDataClient.getStatus({ + namespace: 'default', + }); + + expect(status).toEqual({ + isMaxAmountOfRiskEnginesReached: false, + riskEngineStatus: 'NOT_INSTALLED', + legacyRiskEngineStatus: 'ENABLED', + }); + + esClient.transform.getTransformStats.mockReset(); + }); + }); + }); + + describe('#getConfiguration', () => { + it('retrieves configuration from the saved object', async () => { + mockSavedObjectClient.find.mockResolvedValueOnce(getSavedObjectConfiguration()); + + const configuration = await riskEngineDataClient.getConfiguration(); + + expect(mockSavedObjectClient.find).toHaveBeenCalledTimes(1); + + expect(configuration).toEqual({ + enabled: false, + }); + }); + }); + + describe('enableRiskEngine', () => { + let mockTaskManagerStart: ReturnType; + + beforeEach(() => { + mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); + mockTaskManagerStart = taskManagerMock.createStart(); + }); + + it('returns an error if saved object does not exist', async () => { + mockSavedObjectClient.find.mockResolvedValue({ + page: 1, + per_page: 20, + total: 0, + saved_objects: [], + }); + + await expect( + riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }) + ).rejects.toThrow('Risk engine configuration not found'); + }); + + it('should update saved object attribute', async () => { + await riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }); + + expect(mockSavedObjectClient.update).toHaveBeenCalledWith( + 'risk-engine-configuration', + 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', + { + enabled: true, + }, + { + refresh: 'wait_for', + } + ); + }); + + describe('if task manager throws an error', () => { + beforeEach(() => { + mockTaskManagerStart.ensureScheduled.mockRejectedValueOnce( + new Error('Task Manager error') + ); + }); + + it('disables the risk engine and re-throws the error', async () => { + await expect( + riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }) + ).rejects.toThrow('Task Manager error'); + + expect(mockSavedObjectClient.update).toHaveBeenCalledWith( + 'risk-engine-configuration', + 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', + { + enabled: false, + }, + { + refresh: 'wait_for', + } + ); + }); + }); + }); + + describe('disableRiskEngine', () => { + let mockTaskManagerStart: ReturnType; + + beforeEach(() => { + mockTaskManagerStart = taskManagerMock.createStart(); + }); + + it('should return error if saved object not exist', async () => { + mockSavedObjectClient.find.mockResolvedValueOnce({ + page: 1, + per_page: 20, + total: 0, + saved_objects: [], + }); + + expect.assertions(1); + try { + await riskEngineDataClient.disableRiskEngine({ taskManager: mockTaskManagerStart }); + } catch (e) { + expect(e.message).toEqual('Risk engine configuration not found'); + } + }); + + it('should update saved object attrubute', async () => { + mockSavedObjectClient.find.mockResolvedValueOnce(getSavedObjectConfiguration()); + + await riskEngineDataClient.disableRiskEngine({ taskManager: mockTaskManagerStart }); + + expect(mockSavedObjectClient.update).toHaveBeenCalledWith( + 'risk-engine-configuration', + 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', + { + enabled: false, + }, + { + refresh: 'wait_for', + } + ); + }); + }); + + describe('init', () => { + let mockTaskManagerStart: ReturnType; + const initializeResourcesMock = jest.spyOn( + RiskEngineDataClient.prototype, + 'initializeResources' + ); + const enableRiskEngineMock = jest.spyOn(RiskEngineDataClient.prototype, 'enableRiskEngine'); + + const disableLegacyRiskEngineMock = jest.spyOn( + RiskEngineDataClient.prototype, + 'disableLegacyRiskEngine' + ); + beforeEach(() => { + mockTaskManagerStart = taskManagerMock.createStart(); + disableLegacyRiskEngineMock.mockImplementation(() => Promise.resolve(true)); + + initializeResourcesMock.mockImplementation(() => { + return Promise.resolve(); + }); + + enableRiskEngineMock.mockImplementation(() => { + return Promise.resolve(getSavedObjectConfiguration().saved_objects[0]); + }); + + jest + .spyOn(savedObjectConfig, 'initSavedObjects') + .mockResolvedValue({} as unknown as SavedObject); + }); + + afterEach(() => { + initializeResourcesMock.mockReset(); + enableRiskEngineMock.mockReset(); + disableLegacyRiskEngineMock.mockReset(); + }); + + it('success', async () => { + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: [], + legacyRiskEngineDisabled: true, + riskEngineConfigurationCreated: true, + riskEngineEnabled: true, + riskEngineResourcesInstalled: true, + }); + }); + + it('should catch error for disableLegacyRiskEngine, but continue', async () => { + disableLegacyRiskEngineMock.mockImplementation(() => { + throw new Error('Error disableLegacyRiskEngineMock'); + }); + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: ['Error disableLegacyRiskEngineMock'], + legacyRiskEngineDisabled: false, + riskEngineConfigurationCreated: true, + riskEngineEnabled: true, + riskEngineResourcesInstalled: true, + }); + }); + + it('should catch error for resource init', async () => { + disableLegacyRiskEngineMock.mockImplementationOnce(() => { + throw new Error('Error disableLegacyRiskEngineMock'); + }); + + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: ['Error disableLegacyRiskEngineMock'], + legacyRiskEngineDisabled: false, + riskEngineConfigurationCreated: true, + riskEngineEnabled: true, + riskEngineResourcesInstalled: true, + }); + }); + + it('should catch error for initializeResources and stop', async () => { + initializeResourcesMock.mockImplementationOnce(() => { + throw new Error('Error initializeResourcesMock'); + }); + + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: ['Error initializeResourcesMock'], + legacyRiskEngineDisabled: true, + riskEngineConfigurationCreated: false, + riskEngineEnabled: false, + riskEngineResourcesInstalled: false, + }); + }); + + it('should catch error for initSavedObjects and stop', async () => { + jest.spyOn(savedObjectConfig, 'initSavedObjects').mockImplementationOnce(() => { + throw new Error('Error initSavedObjects'); + }); + + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: ['Error initSavedObjects'], + legacyRiskEngineDisabled: true, + riskEngineConfigurationCreated: false, + riskEngineEnabled: false, + riskEngineResourcesInstalled: true, + }); + }); + + it('should catch error for enableRiskEngineMock and stop', async () => { + enableRiskEngineMock.mockImplementationOnce(() => { + throw new Error('Error enableRiskEngineMock'); + }); + + const initResult = await riskEngineDataClient.init({ + namespace: 'default', + taskManager: mockTaskManagerStart, + }); + + expect(initResult).toEqual({ + errors: ['Error enableRiskEngineMock'], + legacyRiskEngineDisabled: true, + riskEngineConfigurationCreated: true, + riskEngineEnabled: false, + riskEngineResourcesInstalled: true, + }); + }); + }); + }); + } +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts new file mode 100644 index 0000000000000..185d0ae24c2b4 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -0,0 +1,294 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Metadata } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + createOrUpdateComponentTemplate, + createOrUpdateIndexTemplate, +} from '@kbn/alerting-plugin/server'; +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; +import { AssistantResourceNames } from '../types'; +import { + conversationsFieldMap, + getIndexTemplateAndPattern, + mappingComponentName, + totalFieldsLimit, +} from './lib/conversation_configuration_type'; +import { createConcreteWriteIndex } from './lib/create_concrete_write_index'; +import { DataStreamAdapter } from './lib/create_datastream'; +import { AIAssistantDataClient } from '../ai_assistant_data_client'; +import { + InitializationPromise, + ResourceInstallationHelper, + createResourceInstallationHelper, + errorResult, + successResult, +} from './create_resource_installation_helper'; +import { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; +import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; + +export const ECS_CONTEXT = `ecs`; +function getResourceName(resource: string) { + return `.kibana-elastic-ai-assistant-${resource}`; +} + +export const getComponentTemplateName = (name: string) => `.alerts-${name}-mappings`; + +interface AIAssistantServiceOpts { + logger: Logger; + kibanaVersion: string; + elasticsearchClientPromise: Promise; + dataStreamAdapter: DataStreamAdapter; + taskManager: TaskManagerSetupContract; +} + +export interface CreateAIAssistantClientParams { + logger: Logger; + namespace: string; +} + +export class AIAssistantService { + private dataStreamAdapter: DataStreamAdapter; + private initialized: boolean; + private isInitializing: boolean = false; + private registeredNamespaces: Set = new Set(); + private resourceInitializationHelper: ResourceInstallationHelper; + private commonInitPromise: Promise; + + constructor(private readonly options: AIAssistantServiceOpts) { + this.initialized = false; + this.dataStreamAdapter = options.dataStreamAdapter; + + this.commonInitPromise = this.initializeResources(); + + // Create helper for initializing context-specific resources + this.resourceInitializationHelper = createResourceInstallationHelper( + this.options.logger, + this.commonInitPromise, + this.installAndUpdateNamespaceLevelResources.bind(this) + ); + } + + public async isInitialized() { + return this.initialized; + } + + private readonly resourceNames: AssistantResourceNames = { + componentTemplate: { + conversations: getResourceName('component-template-conversations'), + kb: getResourceName('component-template-kb'), + }, + aliases: { + conversations: getResourceName('conversations'), + kb: getResourceName('kb'), + }, + indexPatterns: { + conversations: getResourceName('conversations*'), + kb: getResourceName('kb*'), + }, + indexTemplate: { + conversations: getResourceName('index-template-conversations'), + kb: getResourceName('index-template-kb'), + }, + pipelines: { + kb: getResourceName('kb-ingest-pipeline'), + }, + }; + + public async createAIAssistantDatastreamClient( + opts: CreateAIAssistantClientParams + ): Promise { + // Check if context specific installation has succeeded + const { result: initialized, error } = await this.getResourcesInitializationPromise( + opts.namespace + ); + + // If initialization failed, retry + if (!initialized && error) { + let initPromise: Promise | undefined; + + // If !this.initialized, we know that resource initialization failed + // and we need to retry this before retrying the namespace specific resources + if (!this.initialized) { + if (!this.isInitializing) { + this.options.logger.info(`Retrying common resource initialization`); + initPromise = this.initializeResources(); + } else { + this.options.logger.info( + `Skipped retrying common resource initialization because it is already being retried.` + ); + } + } + + this.resourceInitializationHelper.retry(opts.namespace, initPromise); + + const retryResult = await this.resourceInitializationHelper.getInitializedResources( + opts.namespace ?? DEFAULT_NAMESPACE_STRING + ); + + if (!retryResult.result) { + const errorLogPrefix = `There was an error in the framework installing namespace-level resources and creating concrete indices for namespace "${opts.namespace}" - `; + // Retry also failed + this.options.logger.warn( + retryResult.error && error + ? `${errorLogPrefix}Retry failed with errors: ${error}` + : `${errorLogPrefix}Original error: ${error}; Error after retry: ${retryResult.error}` + ); + return null; + } else { + this.options.logger.info( + `Resource installation for "${opts.namespace}" succeeded after retry` + ); + } + } + + return new AIAssistantDataClient({ + logger: this.options.logger, + elasticsearchClientPromise: this.options.elasticsearchClientPromise, + namespace: opts.namespace, + kibanaVersion: this.options.kibanaVersion, + indexPatternsResorceName: this.resourceNames.indexPatterns.conversations, + }); + } + + public async getResourcesInitializationPromise( + namespace?: string + ): Promise { + const registeredOpts = namespace && this.registeredNamespaces.has(namespace) ? namespace : null; + + if (!registeredOpts) { + const errMsg = `Error getting initialized status for namespace ${namespace} - namespace has not been registered.`; + this.options.logger.error(errMsg); + return errorResult(errMsg); + } + + const result = await this.resourceInitializationHelper.getInitializedResources( + namespace ?? DEFAULT_NAMESPACE_STRING + ); + + // If the context is unrecognized and namespace is not the default, we + // need to kick off resource installation and return the promise + if ( + result.error && + result.error.includes(`Unrecognized context`) && + namespace !== DEFAULT_NAMESPACE_STRING + ) { + this.resourceInitializationHelper.add(namespace); + + return this.resourceInitializationHelper.getInitializedResources(namespace ?? 'default'); + } + + return result; + } + + private async initializeResources(): Promise { + try { + this.options.logger.debug(`Initializing resources for AIAssistantService`); + const esClient = await this.options.elasticsearchClientPromise; + + // TODO: add DLM policy + await Promise.all([ + createOrUpdateComponentTemplate({ + logger: this.options.logger, + esClient, + template: getComponentTemplateFromFieldMap({ + name: `${this.resourceNames.componentTemplate.conversations}-ecs`, + fieldMap: ecsFieldMap, + dynamic: false, + includeSettings: true, + }), + totalFieldsLimit, + }), + createOrUpdateComponentTemplate({ + logger: this.options.logger, + esClient, + template: { + name: this.resourceNames.componentTemplate.conversations, + _meta: { + managed: true, + }, + template: { + settings: {}, + mappings: mappingFromFieldMap(conversationsFieldMap, 'strict'), + }, + } as ClusterPutComponentTemplateRequest, + totalFieldsLimit, + }), + ]); + + this.initialized = true; + this.isInitializing = false; + return successResult(); + } catch (error) { + this.options.logger.error(`Error initializing AI assistant resources: ${error.message}`); + this.initialized = false; + this.isInitializing = false; + return errorResult(error.message); + } + } + + private async installAndUpdateNamespaceLevelResources(namespace?: string) { + try { + this.options.logger.debug(`Initializing namespace level resources for AIAssistantService`); + const esClient = await this.options.elasticsearchClientPromise; + + const indexMetadata: Metadata = { + kibana: { + version: this.options.kibanaVersion, + }, + managed: true, + namespace, + }; + + const indexPatterns = getIndexTemplateAndPattern( + this.resourceNames.indexPatterns.conversations, + namespace ?? 'default' + ); + + await createOrUpdateIndexTemplate({ + logger: this.options.logger, + esClient, + template: { + name: indexPatterns.template, + body: { + data_stream: { hidden: true }, + index_patterns: [indexPatterns.alias], + composed_of: [mappingComponentName], + template: { + lifecycle: {}, + settings: { + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: indexMetadata, + }, + }, + _meta: indexMetadata, + }, + }, + }); + + await createConcreteWriteIndex({ + logger: this.options.logger, + esClient, + totalFieldsLimit, + indexPatterns, + dataStreamAdapter: this.dataStreamAdapter, + }); + } catch (error) { + this.options.logger.error( + `Error initializing AI assistant namespace level resources: ${error.message}` + ); + throw error; + } + } +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts new file mode 100644 index 0000000000000..c64b98d3f7c9e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { FieldMap } from '@kbn/alerts-as-data-utils'; +import { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { IIndexPatternString } from '../../types'; + +const keyword = { + type: 'keyword' as const, + ignore_above: 1024, +}; + +const text = { + type: 'text' as const, +}; + +const date = { + type: 'date' as const, +}; + +const dynamic = { + type: 'object' as const, + dynamic: true, +}; + +const commonFields: ClusterComponentTemplate['component_template']['template'] = { + mappings: { + dynamic_templates: [ + { + numeric_labels: { + path_match: 'numeric_labels.*', + mapping: { + scaling_factor: 1000000, + type: 'scaled_float', + }, + }, + }, + ], + dynamic: false, + properties: { + '@timestamp': date, + labels: dynamic, + numeric_labels: dynamic, + user: { + properties: { + id: keyword, + name: keyword, + }, + }, + conversation: { + properties: { + id: keyword, + title: text, + last_updated: date, + }, + }, + api_config: { + properties: { + connectorId: keyword, + connectorTypeTitle: text, + model: keyword, + provider: keyword, + }, + }, + anonymized_fields: { + type: 'object', + properties: { + field_name: keyword, + value: { + type: 'object', + enabled: false, + }, + uuid: keyword, + }, + }, + namespace: keyword, + messages: { + type: 'object', + properties: { + '@timestamp': date, + message: { + type: 'object', + properties: { + content: text, + event: text, + role: keyword, + }, + }, + }, + }, + public: { + type: 'boolean', + }, + }, + }, +}; + +export const conversationsFieldMap: FieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: false, + }, + 'user.id': { + type: 'keyword', + array: false, + required: true, + }, + 'user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'conversation.id': { + type: 'keyword', + array: false, + required: false, + }, + 'conversation.title': { + type: 'object', + array: false, + required: false, + }, + 'conversation.last_updated': { + type: 'object', + array: false, + required: false, + }, + messages: { + type: 'object', + array: true, + required: false, + }, + 'messages.id': { + type: 'keyword', + array: false, + required: true, + }, + 'messages.role': { + type: 'keyword', + array: false, + required: true, + }, + 'messages.event': { + type: 'keyword', + array: false, + required: true, + }, + 'messages.content': { + type: 'keyword', + array: false, + required: false, + }, + 'messages.anonymized_fields': { + type: 'object', + array: true, + required: false, + }, +} as const; + +export const mappingComponentName = '.conversations-mappings'; +export const totalFieldsLimit = 1000; + +export const getIndexTemplateAndPattern = ( + context: string, + namespace?: string +): IIndexPatternString => { + const concreteNamespace = namespace ? namespace : DEFAULT_NAMESPACE_STRING; + const pattern = `${context}`; + const patternWithNamespace = `${pattern}-${concreteNamespace}`; + return { + template: `${patternWithNamespace}-index-template`, + pattern: `.internal.${patternWithNamespace}-*`, + basePattern: `.${pattern}-*`, + name: `.internal.${patternWithNamespace}-000001`, + alias: `.${patternWithNamespace}`, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts new file mode 100644 index 0000000000000..1d6555ea799a1 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts @@ -0,0 +1,718 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { createConcreteWriteIndex } from './create_concrete_write_index'; +import { getDataStreamAdapter } from './data_stream_adapter'; + +const randomDelayMultiplier = 0.01; +const logger = loggingSystemMock.createLogger(); +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +interface EsError extends Error { + statusCode?: number; + meta?: { + body: { + error: { + type: string; + }; + }; + }; +} + +const GetAliasResponse = { + '.internal.alerts-test.alerts-default-000001': { + aliases: { + alias_1: { + is_hidden: true, + }, + alias_2: { + is_hidden: true, + }, + }, + }, +}; + +const GetDataStreamResponse = { + data_streams: ['any-content-here-means-already-exists'], +} as unknown as IndicesGetDataStreamResponse; + +const SimulateTemplateResponse = { + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, +}; + +const IndexPatterns = { + template: '.alerts-test.alerts-default-index-template', + pattern: '.internal.alerts-test.alerts-default-*', + basePattern: '.alerts-test.alerts-*', + alias: '.alerts-test.alerts-default', + name: '.internal.alerts-test.alerts-default-000001', + validPrefixes: ['.internal.alerts-', '.alerts-'], +}; + +describe('createConcreteWriteIndex', () => { + for (const useDataStream of [false, true]) { + const label = useDataStream ? 'data streams' : 'aliases'; + const dataStreamAdapter = getDataStreamAdapter({ useDataStreamForAlerts: useDataStream }); + + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + describe(`using ${label} for alert indices`, () => { + it(`should call esClient to put index template`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({ + name: '.alerts-test.alerts-default', + }); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + } + }); + + it(`should retry on transient ES errors`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); + clusterClient.indices.create + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ + index: '.internal.alerts-test.alerts-default-000001', + shards_acknowledged: true, + acknowledged: true, + }); + clusterClient.indices.createDataStream + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ + acknowledged: true, + }); + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); + } + }); + + it(`should log and throw error if max retries exceeded`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); + clusterClient.indices.create.mockRejectedValue(new EsErrors.ConnectionError('foo')); + clusterClient.indices.createDataStream.mockRejectedValue( + new EsErrors.ConnectionError('foo') + ); + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Error creating data stream .alerts-test.alerts-default - foo` + : `Error creating concrete write index - foo` + ); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(4); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledTimes(4); + } + }); + + it(`should log and throw error if ES throws error`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); + clusterClient.indices.create.mockRejectedValueOnce(new Error('generic error')); + clusterClient.indices.createDataStream.mockRejectedValueOnce(new Error('generic error')); + + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Error creating data stream .alerts-test.alerts-default - generic error` + : `Error creating concrete write index - generic error` + ); + }); + + it(`should log and return if ES throws resource_already_exists_exception error and existing index is already write index`, async () => { + if (useDataStream) return; + + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, + }, + })); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + }); + + it(`should retry getting index on transient ES error`, async () => { + if (useDataStream) return; + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + const error = new Error(`fail`) as EsError; + error.statusCode = 404; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, + }, + })); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(clusterClient.indices.get).toHaveBeenCalledTimes(3); + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + }); + + it(`should log and throw error if ES throws resource_already_exists_exception error and existing index is not the write index`, async () => { + if (useDataStream) return; + + clusterClient.indices.getAlias.mockImplementation(async () => ({})); + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: false } }, + }, + })); + + const ccwiPromise = createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + await expect(() => ccwiPromise).rejects.toThrowErrorMatchingInlineSnapshot( + `"Attempted to create index: .internal.alerts-test.alerts-default-000001 as the write index for alias: .alerts-test.alerts-default, but the index already exists and is not the write index for the alias"` + ); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + }); + + it(`should call esClient to put index template if get alias throws 404`, async () => { + const error = new Error(`not found`) as EsError; + error.statusCode = 404; + clusterClient.indices.getAlias.mockRejectedValueOnce(error); + clusterClient.indices.getDataStream.mockRejectedValueOnce(error); + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({ + name: '.alerts-test.alerts-default', + }); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + } + }); + + it(`should log and throw error if get alias throws non-404 error`, async () => { + const error = new Error(`fatal error`) as EsError; + error.statusCode = 500; + clusterClient.indices.getAlias.mockRejectedValueOnce(error); + clusterClient.indices.getDataStream.mockRejectedValueOnce(error); + + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"fatal error"`); + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Error fetching data stream for .alerts-test.alerts-default - fatal error` + : `Error fetching concrete indices for .internal.alerts-test.alerts-default-* pattern - fatal error` + ); + }); + + it(`should update underlying settings and mappings of existing concrete indices if they exist`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (!useDataStream) { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + } + + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 1 : 2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 1 : 2); + }); + + it(`should skip updating underlying settings and mappings of existing concrete indices if they follow an unexpected naming convention`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => ({ + bad_index_name: { + aliases: { + alias_1: { + is_hidden: true, + }, + }, + }, + })); + + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (!useDataStream) { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + expect(logger.warn).toHaveBeenCalledWith( + `Found unexpected concrete index name "bad_index_name" while expecting index with one of the following prefixes: [.internal.alerts-,.alerts-] Not updating mappings or settings for this index.` + ); + } + + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 1 : 0); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 1 : 0); + }); + + it(`should retry simulateIndexTemplate on transient ES errors`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementation(async () => SimulateTemplateResponse); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( + useDataStream ? 3 : 4 + ); + }); + + it(`should retry getting alias on transient ES errors`, async () => { + clusterClient.indices.getAlias + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + if (useDataStream) { + expect(clusterClient.indices.getDataStream).toHaveBeenCalledTimes(3); + } else { + expect(clusterClient.indices.getAlias).toHaveBeenCalledTimes(3); + } + }); + + it(`should retry settings update on transient ES errors`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putSettings + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 3 : 4); + }); + + it(`should log and throw error on settings update if max retries exceeded`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putSettings.mockRejectedValue(new EsErrors.ConnectionError('foo')); + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 4 : 7); + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: foo` + : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: foo` + ); + }); + + it(`should log and throw error on settings update if ES throws error`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putSettings.mockRejectedValue(new Error('generic error')); + + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: generic error` + : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: generic error` + ); + }); + + it(`should retry mappings update on transient ES errors`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putMapping + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 3 : 4); + }); + + it(`should log and throw error on mappings update if max retries exceeded`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putMapping.mockRejectedValue(new EsErrors.ConnectionError('foo')); + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 4 : 7); + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Failed to PUT mapping for .alerts-test.alerts-default: foo` + : `Failed to PUT mapping for alias_1: foo` + ); + }); + + it(`should log and throw error on mappings update if ES throws error`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.putMapping.mockRejectedValue(new Error('generic error')); + + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Failed to PUT mapping for .alerts-test.alerts-default: generic error` + : `Failed to PUT mapping for alias_1: generic error` + ); + }); + + it(`should log and return when simulating updated mappings throws error`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Ignored PUT mappings for .alerts-test.alerts-default; error generating simulated mappings: fail` + : `Ignored PUT mappings for alias_1; error generating simulated mappings: fail` + ); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + } + }); + + it(`should log and return when simulating updated mappings returns null`, async () => { + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { ...SimulateTemplateResponse.template, mappings: null }, + })); + + await createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }); + + expect(logger.error).toHaveBeenCalledWith( + useDataStream + ? `Ignored PUT mappings for .alerts-test.alerts-default; simulated mappings were empty` + : `Ignored PUT mappings for alias_1; simulated mappings were empty` + ); + + if (useDataStream) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + } + }); + + it(`should throw error when there are concrete indices but none of them are the write index`, async () => { + if (useDataStream) return; + + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-0001': { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: false, + is_hidden: true, + }, + alias_2: { + is_write_index: false, + is_hidden: true, + }, + }, + }, + })); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + + await expect(() => + createConcreteWriteIndex({ + logger, + esClient: clusterClient, + indexPatterns: IndexPatterns, + totalFieldsLimit: 2500, + dataStreamAdapter, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default"` + ); + }); + }); + } +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts new file mode 100644 index 0000000000000..dd14a969f644c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { get } from 'lodash'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; +import { DataStreamAdapter } from './create_data_stream'; +import { IIndexPatternString } from '../types'; + +export interface ConcreteIndexInfo { + index: string; + alias: string; + isWriteIndex: boolean; +} + +interface UpdateIndexMappingsOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + validIndexPrefixes?: string[]; + concreteIndices: ConcreteIndexInfo[]; +} + +interface UpdateIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + concreteIndexInfo: ConcreteIndexInfo; +} + +const updateTotalFieldLimitSetting = async ({ + logger, + esClient, + totalFieldsLimit, + concreteIndexInfo, +}: UpdateIndexOpts) => { + const { index, alias } = concreteIndexInfo; + try { + await retryTransientEsErrors( + () => + esClient.indices.putSettings({ + index, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }), + { logger } + ); + } catch (err) { + logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for ${alias}: ${err.message}` + ); + throw err; + } +}; + +// This will update the mappings of backing indices but *not* the settings. This +// is due to the fact settings can be classed as dynamic and static, and static +// updates will fail on an index that isn't closed. New settings *will* be applied as part +// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 +const updateUnderlyingMapping = async ({ + logger, + esClient, + concreteIndexInfo, +}: UpdateIndexOpts) => { + const { index, alias } = concreteIndexInfo; + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await retryTransientEsErrors( + () => esClient.indices.simulateIndexTemplate({ name: index }), + { logger } + ); + } catch (err) { + logger.error( + `Ignored PUT mappings for ${alias}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + logger.error(`Ignored PUT mappings for ${alias}; simulated mappings were empty`); + return; + } + + try { + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index, body: simulatedMapping }), + { logger } + ); + } catch (err) { + logger.error(`Failed to PUT mapping for ${alias}: ${err.message}`); + throw err; + } +}; +/** + * Updates the underlying mapping for any existing concrete indices + */ +export const updateIndexMappings = async ({ + logger, + esClient, + totalFieldsLimit, + concreteIndices, + validIndexPrefixes, +}: UpdateIndexMappingsOpts) => { + let validConcreteIndices = []; + if (validIndexPrefixes) { + for (const cIdx of concreteIndices) { + if (!validIndexPrefixes?.some((prefix: string) => cIdx.index.startsWith(prefix))) { + logger.warn( + `Found unexpected concrete index name "${ + cIdx.index + }" while expecting index with one of the following prefixes: [${validIndexPrefixes.join( + ',' + )}] Not updating mappings or settings for this index.` + ); + } else { + validConcreteIndices.push(cIdx); + } + } + } else { + validConcreteIndices = concreteIndices; + } + + logger.debug( + `Updating underlying mappings for ${validConcreteIndices.length} indices / data streams.` + ); + + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + validConcreteIndices.map((index) => + updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) + ) + ); + + // Update mappings of the found indices. + await Promise.all( + validConcreteIndices.map((index) => + updateUnderlyingMapping({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) + ) + ); +}; + +export interface CreateConcreteWriteIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + indexPatterns: IIndexPatternString; + dataStreamAdapter: DataStreamAdapter; +} +/** + * Installs index template that uses installed component template + * Prior to installation, simulates the installation to check for possible + * conflicts. Simulate should return an empty mapping if a template + * conflicts with an already installed template. + */ +export const createConcreteWriteIndex = async (opts: CreateConcreteWriteIndexOpts) => { + await opts.dataStreamAdapter.createStream(opts); +}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts new file mode 100644 index 0000000000000..6a7ac2ab6df65 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStreamAdapter } from './create_datastream'; + +export function createDataStreamAdapterMock(): DataStreamAdapter { + return { + getIndexTemplateFields: jest.fn().mockReturnValue({ + index_patterns: ['index-pattern'], + }), + createStream: jest.fn(), + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts new file mode 100644 index 0000000000000..f3d87d2d3c5dd --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CreateConcreteWriteIndexOpts, updateIndexMappings } from './create_concrete_write_index'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +export interface DataStreamAdapter { + getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields; + createStream(opts: CreateConcreteWriteIndexOpts): Promise; +} + +export interface BulkOpProperties { + require_alias: boolean; +} + +export interface IndexTemplateFields { + data_stream?: { hidden: true }; + index_patterns: string[]; + rollover_alias?: string; +} + +export function getDataStreamAdapter(): DataStreamAdapter { + return new DataStreamImplementation(); +} + +// implementation using data streams +class DataStreamImplementation implements DataStreamAdapter { + getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields { + return { + data_stream: { hidden: true }, + index_patterns: [alias], + }; + } + + async createStream(opts: CreateConcreteWriteIndexOpts): Promise { + return createDataStream(opts); + } +} + +async function createDataStream(opts: CreateConcreteWriteIndexOpts): Promise { + const { logger, esClient, indexPatterns, totalFieldsLimit } = opts; + logger.info(`Creating data stream - ${indexPatterns.alias}`); + + // check if data stream exists + let dataStreamExists = false; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name: indexPatterns.alias, expand_wildcards: 'all' }), + { logger } + ); + dataStreamExists = response.data_streams.length > 0; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${indexPatterns.alias} - ${error.message}`); + throw error; + } + } + + // if a data stream exists, update the underlying mapping + if (dataStreamExists) { + await updateIndexMappings({ + logger, + esClient, + totalFieldsLimit, + concreteIndices: [ + { alias: indexPatterns.alias, index: indexPatterns.alias, isWriteIndex: true }, + ], + }); + } else { + try { + await retryTransientEsErrors( + () => + esClient.indices.createDataStream({ + name: indexPatterns.alias, + }), + { logger } + ); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating data stream ${indexPatterns.alias} - ${error.message}`); + throw error; + } + } + } +} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts new file mode 100644 index 0000000000000..689e81a5cc08e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { loggingSystemMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { getRiskInputsIndex } from './get_risk_inputs_index'; +import { buildDataViewResponseMock } from './get_risk_inputs_index.mock'; + +describe('getRiskInputsIndex', () => { + let soClient: SavedObjectsClientContract; + let logger: Logger; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + logger = loggingSystemMock.create().get('security_solution'); + }); + + it('returns an index and runtimeMappings for an existing dataView', async () => { + (soClient.get as jest.Mock).mockResolvedValueOnce(buildDataViewResponseMock()); + const { + id, + attributes: { runtimeFieldMap, title }, + } = buildDataViewResponseMock(); + + const { index, runtimeMappings } = await getRiskInputsIndex({ + dataViewId: id, + logger, + soClient, + }); + + expect(index).toEqual(title); + expect(runtimeMappings).toEqual(JSON.parse(runtimeFieldMap as string)); + }); + + it('returns the index and empty runtimeMappings for a nonexistent dataView', async () => { + const { index, runtimeMappings } = await getRiskInputsIndex({ + dataViewId: 'my-data-view', + logger, + soClient, + }); + expect(index).toEqual('my-data-view'); + expect(runtimeMappings).toEqual({}); + }); + + it('logs that the dataview was not found', async () => { + await getRiskInputsIndex({ + dataViewId: 'my-data-view', + logger, + soClient, + }); + expect(logger.info).toHaveBeenCalledWith( + "No dataview found for ID 'my-data-view'; using ID instead as simple index pattern" + ); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts new file mode 100644 index 0000000000000..cfd66e63159a2 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import type { DataViewAttributes } from '@kbn/data-views-plugin/common'; + +export interface ConfigurationResponse { + index: string; + runtimeMappings: MappingRuntimeFields; +} + +export const getConfigurationIndex = async ({ + dataViewId, + logger, + soClient, +}: { + dataViewId: string; + logger: Logger; + soClient: SavedObjectsClientContract; +}): Promise => { + try { + const dataView = await soClient.get('index-pattern', dataViewId); + const index = dataView.attributes.title; + const runtimeMappings = + dataView.attributes.runtimeFieldMap != null + ? JSON.parse(dataView.attributes.runtimeFieldMap) + : {}; + + return { + index, + runtimeMappings, + }; + } catch (e) { + logger.info( + `No dataview found for ID '${dataViewId}'; using ID instead as simple index pattern` + ); + return { index: dataViewId, runtimeMappings: {} }; + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts new file mode 100644 index 0000000000000..37927cf531c93 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const elasticAssistantBaseIndexName = 'elastic-assistant'; + +export const allAssistantIndexPattern = '.ds-elastic-assistant*'; + +export const latestAssistantIndexPattern = 'elastic-assistant.elastic-assistant-latest-*'; + +export const getAssistantLatestIndex = (spaceId = 'default') => + `${elasticAssistantBaseIndexName}.elastic-assistant-latest-${spaceId}`; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts new file mode 100644 index 0000000000000..7a3839ad3c5bc --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +const MAX_ATTEMPTS = 3; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: Error) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { + logger, + attempt = 0, + }: { + logger: Logger; + attempt?: number; + } +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... + + logger.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + // delay with some randomness + await delay(retryDelaySec * 1000 * Math.random()); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index 827b428c97803..f2238e9fea3b4 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -7,17 +7,16 @@ import { PluginInitializerContext, - CoreSetup, CoreStart, Plugin, Logger, - IContextProvider, KibanaRequest, SavedObjectsClientContract, } from '@kbn/core/server'; import { once } from 'lodash'; import { + ElasticAssistantPluginCoreSetupDependencies, ElasticAssistantPluginSetup, ElasticAssistantPluginSetupDependencies, ElasticAssistantPluginStart, @@ -32,6 +31,14 @@ import { postEvaluateRoute, postKnowledgeBaseRoute, } from './routes'; +import { AIAssistantService } from './ai_assistant_service'; +import { assistantPromptsType, assistantAnonimizationFieldsType } from './saved_object'; +import { + DataStreamAdapter, + getDataStreamAdapter, +} from './ai_assistant_service/lib/create_datastream'; +import { RequestContextFactory } from './routes/request_context_factory'; +import { PLUGIN_ID } from '../common/constants'; export class ElasticAssistantPlugin implements @@ -43,40 +50,51 @@ export class ElasticAssistantPlugin > { private readonly logger: Logger; + private assistantService: AIAssistantService | undefined; + private dataStreamAdapter: DataStreamAdapter; + private readonly kibanaVersion: PluginInitializerContext['env']['packageInfo']['version']; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); + this.dataStreamAdapter = getDataStreamAdapter(); + this.kibanaVersion = initializerContext.env.packageInfo.version; } - private createRouteHandlerContext = ( - core: CoreSetup, - logger: Logger - ): IContextProvider => { - return async function elasticAssistantRouteHandlerContext(context, request) { - const [_, pluginsStart] = await core.getStartServices(); - - return { - actions: pluginsStart.actions, - logger, - }; - }; - }; - - public setup(core: CoreSetup, plugins: ElasticAssistantPluginSetupDependencies) { + public setup( + core: ElasticAssistantPluginCoreSetupDependencies, + plugins: ElasticAssistantPluginSetupDependencies + ) { this.logger.debug('elasticAssistant: Setup'); - const router = core.http.createRouter(); - core.http.registerRouteHandlerContext< - ElasticAssistantRequestHandlerContext, - 'elasticAssistant' - >( - 'elasticAssistant', - this.createRouteHandlerContext( - core as CoreSetup, - this.logger - ) + this.assistantService = new AIAssistantService({ + logger: this.logger.get('service'), + taskManager: plugins.taskManager, + kibanaVersion: this.kibanaVersion, + dataStreamAdapter: this.dataStreamAdapter, + elasticsearchClientPromise: core + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + }); + + const requestContextFactory = new RequestContextFactory({ + logger: this.logger, + core, + plugins, + kibanaVersion: this.kibanaVersion, + assistantService: this.assistantService, + }); + + const router = core.http.createRouter(); + core.http.registerRouteHandlerContext( + PLUGIN_ID, + (context, request) => requestContextFactory.create(context, request) ); + core.savedObjects.registerType(assistantPromptsType); + core.savedObjects.registerType(assistantAnonimizationFieldsType); + + // this.assistantService registerKBTask + const getElserId: GetElser = once( async (request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) => { return (await plugins.ml.trainedModelsProvider(request, savedObjectsClient).getELSER()) @@ -92,6 +110,8 @@ export class ElasticAssistantPlugin postActionsConnectorExecuteRoute(router, getElserId); // Evaluate postEvaluateRoute(router, getElserId); + // Conversations + return { actions: plugins.actions, }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts new file mode 100644 index 0000000000000..761552644feed --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { memoize } from 'lodash'; + +import type { Logger, KibanaRequest, RequestHandlerContext } from '@kbn/core/server'; + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { + ElasticAssistantApiRequestHandlerContext, + ElasticAssistantPluginCoreSetupDependencies, + ElasticAssistantPluginSetupDependencies, + ElasticAssistantRequestHandlerContext, +} from '../types'; +import { AIAssistantSOClient } from '../saved_object/ai_assistant_so_client'; +import { AIAssistantService } from '../ai_assistant_service'; + +export interface IRequestContextFactory { + create( + context: RequestHandlerContext, + request: KibanaRequest + ): Promise; +} + +interface ConstructorOptions { + logger: Logger; + core: ElasticAssistantPluginCoreSetupDependencies; + plugins: ElasticAssistantPluginSetupDependencies; + kibanaVersion: string; + assistantService: AIAssistantService; +} + +export class RequestContextFactory implements IRequestContextFactory { + private readonly logger: Logger; + private readonly assistantService: AIAssistantService; + + constructor(private readonly options: ConstructorOptions) { + this.logger = options.logger; + this.assistantService = options.assistantService; + } + + public async create( + context: Omit, + request: KibanaRequest + ): Promise { + const { options } = this; + const { core } = options; + + const [, startPlugins] = await core.getStartServices(); + const coreContext = await context.core; + + const getSpaceId = (): string => + startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_NAMESPACE_STRING; + + return { + core: coreContext, + + actions: startPlugins.actions, + + logger: this.logger, + + getServerBasePath: () => core.http.basePath.serverBasePath, + + getSpaceId, + + getAIAssistantSOClient: memoize(() => { + const username = + startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; + return new AIAssistantSOClient({ + logger: options.logger, + user: username, + savedObjectsClient: coreContext.savedObjects.client, + }); + }), + + getAIAssistantDataClient: memoize(async () => + this.assistantService.createAIAssistantDatastreamClient({ + namespace: getSpaceId(), + logger: this.logger, + }) + ), + }; + } +} diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts new file mode 100644 index 0000000000000..c62fcf5495fad --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import { assistantPromptsTypeName } from '.'; +import { AIAssistantPrompts } from '../types'; + +export interface SavedObjectsClientArg { + savedObjectsClient: SavedObjectsClientContract; + namespace: string; +} + +const getDefaultAssistantPrompts = ({ namespace }: { namespace: string }): AIAssistantPrompts[] => [ + { id }, +]; + +export const initSavedObjects = async ({ + namespace, + savedObjectsClient, +}: SavedObjectsClientArg & { namespace: string }) => { + const configuration = await getConfigurationSavedObject({ savedObjectsClient }); + if (configuration) { + return configuration; + } + const result = await savedObjectsClient.bulkCreate( + assistantPromptsTypeName, + getDefaultAssistantPrompts({ namespace }), + {} + ); + + const formattedItems = items.map((item) => { + const savedObjectType = getSavedObjectType({ namespaceType: item.namespace_type ?? 'single' }); + const dateNow = new Date().toISOString(); + + return { + attributes: { + comments: [], + created_at: dateNow, + created_by: user, + description: item.description, + entries: item.entries, + name: item.name, + os_types: item.os_types, + tags: item.tags, + tie_breaker_id: tieBreaker ?? uuidv4(), + type: item.type, + updated_by: user, + version: undefined, + }, + type: savedObjectType, + } as { attributes: ExceptionListSoSchema; type: SavedObjectType }; + }); + + const { saved_objects: savedObjects } = + await savedObjectsClient.bulkCreate(formattedItems); + + return result; +}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts new file mode 100644 index 0000000000000..ce8c5704a25b0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts @@ -0,0 +1,267 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Logger, + SavedObjectsErrorHelpers, + type SavedObjectsClientContract, +} from '@kbn/core/server'; + +import { SortResults } from '@elastic/elasticsearch/lib/api/types'; +import { + assistantPromptsTypeName, + transformSavedObjectToAssistantPrompt, + transformSavedObjectUpdateToAssistantPrompt, +} from './elastic_assistant_prompts_type'; +import { + AssistantPromptSchema, + AssistantPromptSoSchema, + FoundAssistantPromptSchema, + transformSavedObjectsToFoundAssistantPrompt, +} from './assistant_prompts_so_schema'; + +export interface AssistantPromptsCreateOptions { + /** The comments of the endpoint list item */ + content: string; + /** The entries of the endpoint list item */ + promptType: string; + /** The entries of the endpoint list item */ + name: string; + /** The entries of the endpoint list item */ + isDefault?: boolean; + /** The entries of the endpoint list item */ + isNewConversationDefault?: boolean; +} + +export interface AssistantPromptsUpdateOptions { + /** The comments of the endpoint list item */ + content: string; + /** The entries of the endpoint list item */ + promptType: string; + /** The entries of the endpoint list item */ + name: string; + /** The entries of the endpoint list item */ + isDefault?: boolean; + /** The entries of the endpoint list item */ + isNewConversationDefault?: boolean; + id: string; + _version: string; +} + +export interface FindAssistantPromptsOptions { + /** The filter to apply in the search */ + filter?: string; + /** How many per page to return */ + perPage: number; + /** The page number or "undefined" if there is no page number to continue from */ + page: number; + /** The search_after parameter if there is one, otherwise "undefined" can be sent in */ + searchAfter?: SortResults; + /** The sort field string if there is one, otherwise "undefined" can be sent in */ + sortField?: string; + /** The sort order of "asc" or "desc", otherwise "undefined" can be sent in */ + sortOrder?: 'asc' | 'desc'; +} + +export interface ConstructorOptions { + /** User creating, modifying, deleting, or updating the prompts */ + user: string; + /** Saved objects client to create, modify, delete, the prompts */ + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +/** + * Class for use for prompts that are used for AI assistant. + */ +export class AIAssistantSOClient { + /** User creating, modifying, deleting, or updating the prompts */ + private readonly user: string; + + /** Saved objects client to create, modify, delete, the prompts */ + private readonly savedObjectsClient: SavedObjectsClientContract; + + /** + * Constructs the assistant client + * @param options + * @param options.user The user associated with the action for exception list + * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI prompts + */ + constructor({ user, savedObjectsClient }: ConstructorOptions) { + this.user = user; + this.savedObjectsClient = savedObjectsClient; + } + + /** + * Fetch an exception list parent container + * @param options + * @param options.id the "id" of an exception list + * @returns The found exception list or null if none exists + */ + public getPrompt = async (id: string): Promise => { + const { savedObjectsClient } = this; + if (id != null) { + try { + const savedObject = await savedObjectsClient.get( + assistantPromptsTypeName, + id + ); + return transformSavedObjectToAssistantPrompt({ savedObject }); + } catch (err) { + if (SavedObjectsErrorHelpers.isNotFoundError(err)) { + return null; + } else { + throw err; + } + } + } else { + return null; + } + }; + + /** + * This creates an agnostic space endpoint list if it does not exist. This tries to be + * as fast as possible by ignoring conflict errors and not returning the contents of the + * list if it already exists. + * @returns AssistantPromptSchema if it created the endpoint list, otherwise null if it already exists + */ + public createPrompt = async ({ + promptType, + content, + name, + isDefault, + isNewConversationDefault, + }: AssistantPromptsCreateOptions): Promise => { + const { savedObjectsClient, user } = this; + + const dateNow = new Date().toISOString(); + try { + const savedObject = await savedObjectsClient.create( + assistantPromptsTypeName, + { + created_at: dateNow, + created_by: user, + content, + name, + is_default: isDefault ?? false, + is_new_conversation_default: isNewConversationDefault ?? false, + prompt_type: promptType, + updated_by: user, + version: 1, + } + ); + return transformSavedObjectToAssistantPrompt({ savedObject }); + } catch (err) { + if (SavedObjectsErrorHelpers.isConflictError(err)) { + return null; + } else { + throw err; + } + } + }; + + /** + * This is the same as "updateExceptionListItem" except it applies specifically to the endpoint list and will + * auto-call the "createEndpointList" for you so that you have the best chance of the endpoint + * being there if it did not exist before. If the list did not exist before, then creating it here will still cause a + * return of null but at least the list exists again. + * @param options + * @param options._version The version to update the endpoint list item to + * @param options.comments The comments of the endpoint list item + * @param options.description The description of the endpoint list item + * @param options.entries The entries of the endpoint list item + * @param options.id The id of the list item (Either this or itemId has to be defined) + * @param options.itemId The item id of the list item (Either this or id has to be defined) + * @param options.meta Optional meta data of the list item + * @param options.name The name of the list item + * @param options.osTypes The OS type of the list item + * @param options.tags Tags of the endpoint list item + * @param options.type The type of the endpoint list item (Default is "simple") + * @returns The exception list item updated, otherwise null if not updated + */ + public updatePromptItem = async ({ + promptType, + content, + name, + isDefault, + isNewConversationDefault, + id, + _version, + }: AssistantPromptsUpdateOptions): Promise => { + const { savedObjectsClient, user } = this; + const prompt = await this.getPrompt(id); + if (prompt == null) { + return null; + } else { + const savedObject = await savedObjectsClient.update( + assistantPromptsTypeName, + prompt.id, + { + content, + is_default: isDefault, + is_new_conversation_default: isNewConversationDefault, + prompt_type: promptType, + name, + updated_by: user, + }, + { + version: _version, + } + ); + return transformSavedObjectUpdateToAssistantPrompt({ + prompt, + savedObject, + }); + } + }; + + /** + * Delete the prompt by id + * @param options + * @param options.id the "id" of the prompt + */ + public deletePromptById = async (id: string): Promise => { + const { savedObjectsClient } = this; + + await savedObjectsClient.delete(assistantPromptsTypeName, id); + }; + + /** + * Finds exception lists given a set of criteria. + * @param options + * @param options.filter The filter to apply in the search + * @param options.perPage How many per page to return + * @param options.page The page number or "undefined" if there is no page number to continue from + * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in + * @param options.searchAfter The search_after parameter if there is one, otherwise "undefined" can be sent in + * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in + * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in + * @returns The found exception lists or null if nothing is found + */ + public findPrompts = async ({ + filter, + perPage, + page, + searchAfter, + sortField, + sortOrder, + }: FindAssistantPromptsOptions): Promise => { + const { savedObjectsClient } = this; + + const savedObjectsFindResponse = await savedObjectsClient.find({ + filter, + page, + perPage, + searchAfter, + sortField, + sortOrder, + type: assistantPromptsTypeName, + }); + + return transformSavedObjectsToFoundAssistantPrompt({ savedObjectsFindResponse }); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts b/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts new file mode 100644 index 0000000000000..5039264cff5c3 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +import { versionOrUndefined } from '@kbn/securitysolution-io-ts-types'; +import { SavedObjectsFindResponse } from '@kbn/core/server'; +import { transformSavedObjectToAssistantPrompt } from './elastic_assistant_prompts_type'; + +export const assistantPromptSoSchema = t.exact( + t.type({ + created_at: t.string, + created_by: t.string, + content: t.string, + is_default: t.boolean, + is_new_conversation_default: t.boolean, + name: t.string, + prompt_type: t.string, + updated_by: t.string, + version: versionOrUndefined, + }) +); + +export type AssistantPromptSoSchema = t.TypeOf; + +export const _version = t.string; +export const _versionOrUndefined = t.union([_version, t.undefined]); + +export const assistantPromptSchema = t.exact( + t.type({ + _version: _versionOrUndefined, + created_at: t.string, + created_by: t.string, + content: t.string, + is_default: t.boolean, + is_new_conversation_default: t.boolean, + name: t.string, + prompt_type: t.string, + updated_by: t.string, + updated_at: t.string, + id: t.string, + version: versionOrUndefined, + }) +); + +export type AssistantPromptSchema = t.TypeOf; + +export const foundAssistantPromptSchema = t.intersection([ + t.exact( + t.type({ + data: t.array(assistantPromptSchema), + page: t.number, + per_page: t.number, + total: t.number, + }) + ), + t.exact(t.partial({})), +]); + +export type FoundAssistantPromptSchema = t.TypeOf; + +export const transformSavedObjectsToFoundAssistantPrompt = ({ + savedObjectsFindResponse, +}: { + savedObjectsFindResponse: SavedObjectsFindResponse; +}): FoundAssistantPromptSchema => { + return { + data: savedObjectsFindResponse.saved_objects.map((savedObject) => + transformSavedObjectToAssistantPrompt({ savedObject }) + ), + page: savedObjectsFindResponse.page, + per_page: savedObjectsFindResponse.per_page, + total: savedObjectsFindResponse.total, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts new file mode 100644 index 0000000000000..fb1c201196974 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import type { SavedObjectsType } from '@kbn/core/server'; + +export const assistantAnonimizationFieldsTypeName = 'elastic-ai-assistant-anonimization-fields'; + +export const assistantAnonimizationFieldsTypeMappings: SavedObjectsType['mappings'] = { + properties: { + fieldId: { + type: 'keyword', + }, + defaultAllow: { + type: 'boolean', + }, + defaultAllowReplacement: { + type: 'boolean', + }, + updated_at: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + }, +}; + +export const assistantAnonimizationFieldsType: SavedObjectsType = { + name: assistantAnonimizationFieldsTypeName, + indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, + hidden: false, + namespaceType: 'multiple-isolated', + mappings: assistantAnonimizationFieldsTypeMappings, +}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts new file mode 100644 index 0000000000000..bf6be78a1fbb0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import type { SavedObject, SavedObjectsType, SavedObjectsUpdateResponse } from '@kbn/core/server'; +import { AssistantPromptSchema, AssistantPromptSoSchema } from './assistant_prompts_so_schema'; + +export const assistantPromptsTypeName = 'elastic-ai-assistant-prompts'; + +export const assistantPromptsTypeMappings: SavedObjectsType['mappings'] = { + properties: { + id: { + type: 'keyword', + }, + isDefault: { + type: 'boolean', + }, + isNewConversationDefault: { + type: 'boolean', + }, + name: { + type: 'keyword', + }, + promptType: { + type: 'keyword', + }, + content: { + type: 'keyword', + }, + updated_at: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + }, +}; + +export const assistantPromptsType: SavedObjectsType = { + name: assistantPromptsTypeName, + indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, + hidden: false, + namespaceType: 'multiple-isolated', + mappings: assistantPromptsTypeMappings, +}; + +export const transformSavedObjectToAssistantPrompt = ({ + savedObject, +}: { + savedObject: SavedObject; +}): AssistantPromptSchema => { + const dateNow = new Date().toISOString(); + const { + version: _version, + attributes: { + /* eslint-disable @typescript-eslint/naming-convention */ + created_at, + created_by, + content, + is_default, + is_new_conversation_default, + prompt_type, + name, + updated_by, + version, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + id, + updated_at: updatedAt, + } = savedObject; + + return { + _version, + created_at, + created_by, + content, + id, + name, + prompt_type, + is_default, + is_new_conversation_default, + updated_at: updatedAt ?? dateNow, + updated_by, + version: version ?? 1, + }; +}; + +export const transformSavedObjectUpdateToAssistantPrompt = ({ + prompt, + savedObject, +}: { + prompt: AssistantPromptSchema; + savedObject: SavedObjectsUpdateResponse; +}): AssistantPromptSchema => { + const dateNow = new Date().toISOString(); + const { + version: _version, + attributes: { name, updated_by: updatedBy, content, prompt_type: promptType, version }, + id, + updated_at: updatedAt, + } = savedObject; + + // TODO: Change this to do a decode and throw if the saved object is not as expected. + // TODO: Do a throw if after the decode this is not the correct "list_type: list" + // TODO: Update exception list and item types (perhaps separating out) so as to avoid + // defaulting + return { + _version, + created_at: prompt.created_at, + created_by: prompt.created_by, + content: content ?? prompt.content, + prompt_type: promptType ?? prompt.prompt_type, + version: version ?? prompt.version, + id, + is_default: prompt.is_default, + is_new_conversation_default: prompt.is_new_conversation_default, + name: name ?? prompt.name, + updated_at: updatedAt ?? dateNow, + updated_by: updatedBy ?? prompt.updated_by, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/index.ts b/x-pack/plugins/elastic_assistant/server/saved_object/index.ts new file mode 100644 index 0000000000000..3362a36721860 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './elastic_assistant_prompts_type'; +export * from './elastic_assistant_anonimization_fields_type'; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml b/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml new file mode 100644 index 0000000000000..20873b1a066da --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml @@ -0,0 +1,439 @@ +openapi: 3.0.0 +info: + version: 1.0.0 + title: Elastic Assistant Conversation API + description: These APIs allow the consumer to manage conversations within Elastic Assistant. +paths: + /conversation: + post: + summary: Trigger calculation of Risk Scores + description: Calculates and persists a segment of Risk Scores, returning details about the calculation. + requestBody: + description: Details about the Risk Scores being calculated + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresCalculationRequest' + required: true + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresCalculationResponse' + '400': + description: Invalid request + /preview: + post: + summary: Preview the calculation of Risk Scores + description: Calculates and returns a list of Risk Scores, sorted by identifier_type and risk score. + requestBody: + description: Details about the Risk Scores being requested + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresPreviewRequest' + required: true + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresPreviewResponse' + '400': + description: Invalid request + /engine/status: + get: + summary: Get the status of the Risk Engine + description: Returns the status of both the legacy transform-based risk engine, as well as the new risk engine + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskEngineStatusResponse' + + /engine/init: + post: + summary: Initialize the Risk Engine + description: Initializes the Risk Engine by creating the necessary indices and mappings, removing old transforms, and starting the new risk engine + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskEngineInitResponse' + /engine/enable: + post: + summary: Enable the Risk Engine + requestBody: + content: + application/json: {} + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskEngineEnableResponse' + /engine/disable: + post: + summary: Disable the Risk Engine + requestBody: + content: + application/json: {} + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskEngineDisableResponse' + + +components: + schemas: + RiskScoresCalculationRequest: + type: object + required: + - data_view_id + - identifier_type + - range + properties: + after_keys: + description: Used to calculate a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. + allOf: + - $ref: '#/components/schemas/AfterKeys' + data_view_id: + $ref: '#/components/schemas/DataViewId' + description: The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. + debug: + description: If set to `true`, the internal ES requests/responses will be logged in Kibana. + type: boolean + filter: + $ref: '#/components/schemas/Filter' + description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores calculated. + page_size: + $ref: '#/components/schemas/PageSize' + identifier_type: + description: Used to restrict the type of risk scores calculated. + allOf: + - $ref: '#/components/schemas/IdentifierType' + range: + $ref: '#/components/schemas/DateRange' + description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. + weights: + $ref: '#/components/schemas/RiskScoreWeights' + + RiskScoresPreviewRequest: + type: object + required: + - data_view_id + properties: + after_keys: + description: Used to retrieve a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. + allOf: + - $ref: '#/components/schemas/AfterKeys' + data_view_id: + $ref: '#/components/schemas/DataViewId' + description: The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. + debug: + description: If set to `true`, a `debug` key is added to the response, containing both the internal request and response with elasticsearch. + type: boolean + filter: + $ref: '#/components/schemas/Filter' + description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores returned. + page_size: + $ref: '#/components/schemas/PageSize' + identifier_type: + description: Used to restrict the type of risk scores involved. If unspecified, both `host` and `user` scores will be returned. + allOf: + - $ref: '#/components/schemas/IdentifierType' + range: + $ref: '#/components/schemas/DateRange' + description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. + weights: + $ref: '#/components/schemas/RiskScoreWeights' + + RiskScoresCalculationResponse: + type: object + required: + - after_keys + - errors + - scores_written + properties: + after_keys: + description: Used to obtain the next "page" of risk scores. See also the `after_keys` key in a risk scores request. If this key is empty, the calculation is complete. + allOf: + - $ref: '#/components/schemas/AfterKeys' + errors: + type: array + description: A list of errors encountered during the calculation. + items: + type: string + scores_written: + type: number + format: integer + description: The number of risk scores persisted to elasticsearch. + + RiskScoresPreviewResponse: + type: object + required: + - after_keys + - scores + properties: + after_keys: + description: Used to obtain the next "page" of risk scores. See also the `after_keys` key in a risk scores request. If this key is empty, the calculation is complete. + allOf: + - $ref: '#/components/schemas/AfterKeys' + debug: + description: Object containing debug information, particularly the internal request and response from elasticsearch + type: object + properties: + request: + type: string + response: + type: string + scores: + type: array + description: A list of risk scores + items: + $ref: '#/components/schemas/RiskScore' + RiskEngineStatusResponse: + type: object + properties: + legacy_risk_engine_status: + $ref: '#/components/schemas/RiskEngineStatus' + risk_engine_status: + $ref: '#/components/schemas/RiskEngineStatus' + is_max_amount_of_risk_engines_reached: + description: Indicates whether the maximum amount of risk engines has been reached + type: boolean + RiskEngineInitResponse: + type: object + properties: + result: + type: object + properties: + risk_engine_enabled: + type: boolean + risk_engine_resources_installed: + type: boolean + risk_engine_configuration_created: + type: boolean + legacy_risk_engine_disabled: + type: boolean + errors: + type: array + items: + type: string + + + + RiskEngineEnableResponse: + type: object + properties: + success: + type: boolean + RiskEngineDisableResponse: + type: object + properties: + success: + type: boolean + + + AfterKeys: + type: object + properties: + host: + type: object + additionalProperties: + type: string + user: + type: object + additionalProperties: + type: string + example: + host: + 'host.name': 'example.host' + user: + 'user.name': 'example_user_name' + DataViewId: + description: The identifier of the Kibana data view to be used when generating risk scores. + example: security-solution-default + type: string + Filter: + description: An elasticsearch DSL filter object. Used to filter the risk inputs involved, which implicitly filters the risk scores themselves. + $ref: 'https://cloud.elastic.co/api/v1/api-docs/spec.json#/definitions/QueryContainer' + PageSize: + description: Specifies how many scores will be involved in a given calculation. Note that this value is per `identifier_type`, i.e. a value of 10 will calculate 10 host scores and 10 user scores, if available. To avoid missed data, keep this value consistent while paginating through scores. + default: 1000 + type: number + DateRange: + description: Defines the time period on which risk inputs will be filtered. + type: object + required: + - start + - end + properties: + start: + $ref: '#/components/schemas/KibanaDate' + end: + $ref: '#/components/schemas/KibanaDate' + KibanaDate: + type: string + oneOf: + - format: date + - format: date-time + - format: datemath + example: '2017-07-21T17:32:28Z' + IdentifierType: + type: string + enum: + - host + - user + RiskScore: + type: object + required: + - '@timestamp' + - id_field + - id_value + - calculated_level + - calculated_score + - calculated_score_norm + - category_1_score + - category_1_count + - inputs + properties: + '@timestamp': + type: string + format: 'date-time' + example: '2017-07-21T17:32:28Z' + description: The time at which the risk score was calculated. + id_field: + type: string + example: 'host.name' + description: The identifier field defining this risk score. Coupled with `id_value`, uniquely identifies the entity being scored. + id_value: + type: string + example: 'example.host' + description: The identifier value defining this risk score. Coupled with `id_field`, uniquely identifies the entity being scored. + calculated_level: + type: string + example: 'Critical' + description: Lexical description of the entity's risk. + calculated_score: + type: number + format: double + description: The raw numeric value of the given entity's risk score. + calculated_score_norm: + type: number + format: double + minimum: 0 + maximum: 100 + description: The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + category_1_score: + type: number + format: double + description: The contribution of Category 1 to the overall risk score (`calculated_score`). Category 1 contains Detection Engine Alerts. + category_1_count: + type: number + format: integer + description: The number of risk input documents that contributed to the Category 1 score (`category_1_score`). + inputs: + type: array + description: A list of the highest-risk documents contributing to this risk score. Useful for investigative purposes. + items: + $ref: '#/components/schemas/RiskScoreInput' + + RiskScoreInput: + description: A generic representation of a document contributing to a Risk Score. + type: object + properties: + id: + type: string + example: 91a93376a507e86cfbf282166275b89f9dbdb1f0be6c8103c6ff2909ca8e1a1c + description: The unique identifier (`_id`) of the original source document + index: + type: string + example: .internal.alerts-security.alerts-default-000001 + description: The unique index (`_index`) of the original source document + category: + type: string + example: category_1 + description: The risk category of the risk input document. + description: + type: string + example: 'Generated from Detection Engine Rule: Malware Prevention Alert' + description: A human-readable description of the risk input document. + risk_score: + type: number + format: double + minimum: 0 + maximum: 100 + description: The weighted risk score of the risk input document. + timestamp: + type: string + example: '2017-07-21T17:32:28Z' + description: The @timestamp of the risk input document. + RiskScoreWeight: + description: "Configuration used to tune risk scoring. Weights can be used to change the score contribution of risk inputs for hosts and users at both a global level and also for Risk Input categories (e.g. 'category_1')." + type: object + required: + - type + properties: + type: + type: string + value: + type: string + host: + type: number + format: double + minimum: 0 + maximum: 1 + user: + type: number + format: double + minimum: 0 + maximum: 1 + example: + type: 'risk_category' + value: 'category_1' + host: 0.8 + user: 0.4 + RiskScoreWeights: + description: 'A list of weights to be applied to the scoring calculation.' + type: array + items: + $ref: '#/components/schemas/RiskScoreWeight' + example: + - type: 'risk_category' + value: 'category_1' + host: 0.8 + user: 0.4 + - type: 'global_identifier' + host: 0.5 + user: 0.1 + RiskEngineStatus: + type: string + enum: + - 'NOT_INSTALLED' + - 'DISABLED' + - 'ENABLED' + RiskEngineInitStep: + type: object + required: + - type + - success + properties: + type: + type: string + success: + type: boolean + error: + type: string + \ No newline at end of file diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index ed9081c084420..cf8c645aeebc2 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -10,12 +10,19 @@ import type { PluginStartContract as ActionsPluginStart, } from '@kbn/actions-plugin/server'; import type { + CoreRequestHandlerContext, + CoreSetup, CustomRequestHandlerContext, KibanaRequest, Logger, SavedObjectsClientContract, } from '@kbn/core/server'; import { type MlPluginSetup } from '@kbn/ml-plugin/server'; +import { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; +import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { AIAssistantSOClient } from './saved_object/ai_assistant_so_client'; +import { AIAssistantDataClient } from './ai_assistant_data_client'; /** The plugin setup interface */ export interface ElasticAssistantPluginSetup { @@ -30,14 +37,23 @@ export interface ElasticAssistantPluginStart { export interface ElasticAssistantPluginSetupDependencies { actions: ActionsPluginSetup; ml: MlPluginSetup; + taskManager: TaskManagerSetupContract; + spaces?: SpacesPluginSetup; } export interface ElasticAssistantPluginStartDependencies { actions: ActionsPluginStart; + spaces?: SpacesPluginStart; + security: SecurityPluginStart; } export interface ElasticAssistantApiRequestHandlerContext { + core: CoreRequestHandlerContext; actions: ActionsPluginStart; logger: Logger; + getServerBasePath: () => string; + getSpaceId: () => string; + getAIAssistantDataClient: () => Promise; + getAIAssistantSOClient: () => AIAssistantSOClient; } /** @@ -47,7 +63,63 @@ export type ElasticAssistantRequestHandlerContext = CustomRequestHandlerContext< elasticAssistant: ElasticAssistantApiRequestHandlerContext; }>; +export type ElasticAssistantPluginCoreSetupDependencies = CoreSetup< + ElasticAssistantPluginStartDependencies, + ElasticAssistantPluginStart +>; + export type GetElser = ( request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract ) => Promise | never; + +export interface InitAssistantResult { + assistantResourcesInstalled: boolean; + assistantNamespaceResourcesInstalled: boolean; + assistantSettingsCreated: boolean; + errors: string[]; +} + +export interface AssistantResourceNames { + componentTemplate: { + conversations: string; + kb: string; + }; + indexTemplate: { + conversations: string; + kb: string; + }; + aliases: { + conversations: string; + kb: string; + }; + indexPatterns: { + conversations: string; + kb: string; + }; + pipelines: { + kb: string; + }; +} + +export interface IIndexPatternString { + template: string; + pattern: string; + alias: string; + name: string; + basePattern: string; + validPrefixes?: string[]; + secondaryAlias?: string; +} + +export interface PublicAIAssistantDataClient { + +} + +export interface IAIAssistantDataClient { + client(): PublicAIAssistantDataClient | null; +} + +export interface AIAssistantPrompts { + id: string; +} From 0aeb11e528ef8c0df6396f2b7c0ff5f4bc4c8ccb Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 17 Dec 2023 21:14:15 -0800 Subject: [PATCH 002/141] [Security solution] AI Assistant persistent storage. --- .../impl/assistant/api/conversations.ts | 221 +++++++++ .../{api.test.tsx => api/index.test.tsx} | 6 +- .../impl/assistant/{api.tsx => api/index.tsx} | 14 +- .../api/use_bulk_actions_conversations.ts | 60 +++ .../api/use_fetch_conversations_by_user.ts | 44 ++ .../assistant/chat_send/use_chat_send.tsx | 8 +- .../conversation_selector/index.tsx | 36 +- .../conversation_selector_settings/index.tsx | 27 +- .../conversation_settings.tsx | 6 +- .../impl/assistant/index.test.tsx | 1 + .../impl/assistant/index.tsx | 25 +- .../conversation_multi_selector.tsx | 2 +- .../system_prompt_settings.tsx | 5 +- .../use_settings_updater.tsx | 60 ++- .../impl/assistant/use_conversation/index.tsx | 331 ++++++------- .../use_conversation/sample_conversations.tsx | 1 + .../impl/assistant_context/index.tsx | 55 +-- .../impl/assistant_context/types.tsx | 6 + .../connector_selector_inline.tsx | 2 + .../connectorland/connector_setup/index.tsx | 30 +- .../impl/mock/conversation.ts | 3 + .../packages/kbn-elastic-assistant/index.ts | 2 + .../elastic_assistant/common/constants.ts | 14 + x-pack/plugins/elastic_assistant/kibana.jsonc | 4 +- .../assistant_data_writer.ts | 88 ---- .../server/ai_assistant_data_client/index.ts | 61 --- .../server/ai_assistant_service/index.ts | 48 +- .../lib/conversation_configuration_type.ts | 176 +++---- .../lib/create_concrete_write_index.ts | 4 +- .../conversations_data_writer.ts | 115 +++++ .../create_conversation.ts | 165 +++++++ .../delete_conversation.ts | 45 ++ .../find_conversations.ts | 89 ++++ .../get_conversation.ts | 34 ++ .../server/conversations_data_client/index.ts | 213 +++++++++ .../conversations_data_client/transforms.ts | 72 +++ .../server/conversations_data_client/types.ts | 45 ++ .../update_conversation.ts | 184 ++++++++ .../server/lib/langchain/types.ts | 2 +- .../server/lib/wait_until_document_indexed.ts | 25 + .../elastic_assistant/server/plugin.ts | 2 + .../routes/conversation/bulk_actions_route.ts | 187 ++++++++ .../routes/conversation/create_route.ts | 76 +++ .../routes/conversation/delete_route.ts | 71 +++ .../server/routes/conversation/find_route.ts | 73 +++ .../find_user_conversations_route.ts | 75 +++ .../server/routes/conversation/read_route.ts | 59 +++ .../routes/conversation/update_route.ts | 85 ++++ .../routes/custom_http_request_error.ts | 15 + .../routes/post_actions_connector_execute.ts | 2 +- .../server/routes/register_routes.ts | 35 ++ .../server/routes/request_context_factory.ts | 14 +- .../server/routes/route_validation.ts | 21 + .../elastic_assistant/server/routes/utils.ts | 164 +++++++ .../post_actions_connector_execute.ts | 0 .../server/schemas/conversation_apis.yml | 439 ------------------ .../bulk_crud_conversations_route.gen.ts | 105 +++++ .../bulk_crud_conversations_route.schema.yaml | 177 +++++++ .../conversations/common_attributes.gen.ts | 235 ++++++++++ .../common_attributes.schema.yaml | 236 ++++++++++ .../crud_conversation_route.schema.yml | 97 ++++ .../find_conversations_route.gen.ts | 65 +++ .../find_conversations_route.schema.yaml | 94 ++++ .../evaluate/post_evaluate_route.schema.yml | 97 ++++ .../knowledge_base/crud_kb_route.schema.yml | 97 ++++ .../plugins/elastic_assistant/server/types.ts | 11 +- .../assistant/content/conversations/index.tsx | 6 + .../assistant/get_comments/index.test.tsx | 1 + .../public/assistant/provider.tsx | 31 +- .../use_assistant_telemetry/index.tsx | 2 +- .../use_conversation_store/index.tsx | 44 +- .../timeline/tabs_content/index.tsx | 2 +- 72 files changed, 3912 insertions(+), 1035 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts rename x-pack/packages/kbn-elastic-assistant/impl/assistant/{api.test.tsx => api/index.test.tsx} (98%) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/{api.tsx => api/index.tsx} (96%) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/custom_http_request_error.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/register_routes.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/route_validation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/utils.ts rename x-pack/plugins/elastic_assistant/server/schemas/{ => actions_connector}/post_actions_connector_execute.ts (100%) delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts new file mode 100644 index 0000000000000..c0eec6ca5599a --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; +import { HttpSetup } from '@kbn/core/public'; +import { IHttpFetchError } from '@kbn/core-http-browser'; +import { Conversation, Message } from '../../assistant_context/types'; + +export interface GetConversationsParams { + http: HttpSetup; + user?: string; + signal?: AbortSignal | undefined; +} + +export interface GetConversationByIdParams { + http: HttpSetup; + id: string; + signal?: AbortSignal | undefined; +} + +/** + * API call for getting current user conversations. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const getUserConversations = async ({ + http, + user, + signal, +}: GetConversationsParams): Promise => { + try { + const path = `/api/elastic_assistant/conversations`; + const response = await http.fetch(path, { + method: 'GET', + query: { + user, + }, + version: '2023-10-31', + signal, + }); + + return response as Conversation[]; + } catch (error) { + return error as IHttpFetchError; + } +}; + +/** + * API call for getting current user conversations. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const getConversationById = async ({ + http, + id, + signal, +}: GetConversationByIdParams): Promise => { + try { + const path = `/api/elastic_assistant/conversations/${id || ''}`; + const response = await http.fetch(path, { + method: 'GET', + version: '2023-10-31', + signal, + }); + + return response as Conversation; + } catch (error) { + return error as IHttpFetchError; + } +}; + +export interface PostConversationParams { + http: HttpSetup; + conversation: Conversation; + signal?: AbortSignal | undefined; +} + +export interface PostConversationResponse { + conversation: Conversation; +} + +/** + * API call for setting up the Knowledge Base. Provide a resource to set up a specific resource. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.resource] - Resource to be added to the KB, otherwise sets up the base KB + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const createConversationApi = async ({ + http, + conversation, + signal, +}: PostConversationParams): Promise => { + try { + const path = `/api/elastic_assistant/conversations`; + const response = await http.post(path, { + body: JSON.stringify(conversation), + version: '2023-10-31', + signal, + }); + + return response as PostConversationResponse; + } catch (error) { + return error as IHttpFetchError; + } +}; + +export interface DeleteConversationParams { + http: HttpSetup; + id: string; + signal?: AbortSignal | undefined; +} + +export interface DeleteConversationResponse { + success: boolean; +} + +/** + * API call for deleting the Knowledge Base. Provide a resource to delete that specific resource. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.id] - Resource to be deleted from the KB, otherwise delete the entire KB + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const deleteConversationApi = async ({ + http, + id, + signal, +}: DeleteConversationParams): Promise => { + try { + const path = `/api/elastic_assistant/conversations/${id || ''}`; + const response = await http.fetch(path, { + method: 'DELETE', + version: '2023-10-31', + signal, + }); + + return response as DeleteConversationResponse; + } catch (error) { + return error as IHttpFetchError; + } +}; + +export interface PutConversationMessageParams { + http: HttpSetup; + conversationId: string; + messages?: Message[]; + apiConfig?: { + connectorId?: string; + connectorTypeTitle?: string; + defaultSystemPromptId?: string; + provider?: OpenAiProviderType; + model?: string; + }; + replacements?: Record; + signal?: AbortSignal | undefined; +} + +export interface PostEvaluationResponse { + evaluationId: string; + success: boolean; +} + +/** + * API call for evaluating models. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.evalParams] - Params necessary for evaluation + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const updateConversationApi = async ({ + http, + conversationId, + messages, + apiConfig, + replacements, + signal, +}: PutConversationMessageParams): Promise => { + try { + const path = `/api/elastic_assistant/conversations/${conversationId || ''}`; + const response = await http.fetch(path, { + method: 'PUT', + body: JSON.stringify({ + id: conversationId, + messages, + replacements, + apiConfig, + }), + headers: { + 'Content-Type': 'application/json', + }, + version: '2023-10-31', + signal, + }); + + return response as Conversation; + } catch (error) { + return error as IHttpFetchError; + } +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx similarity index 98% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx index a4dd40e4f1c4d..ea73d681b3d49 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx @@ -15,9 +15,9 @@ import { getKnowledgeBaseStatus, postEvaluation, postKnowledgeBase, -} from './api'; -import type { Conversation, Message } from '../assistant_context/types'; -import { API_ERROR } from './translations'; +} from '.'; +import type { Conversation, Message } from '../../assistant_context/types'; +import { API_ERROR } from '../translations'; jest.mock('@kbn/core-http-browser'); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx similarity index 96% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index 9b1d3d74035fe..7d03a4bcf5e16 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -6,17 +6,17 @@ */ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/public/common'; - -import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; -import type { Conversation, Message } from '../assistant_context/types'; -import { API_ERROR } from './translations'; -import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector'; +import { HttpSetup } from '@kbn/core/public'; +import { IHttpFetchError } from '@kbn/core-http-browser'; +import type { Conversation, Message } from '../../assistant_context/types'; +import { API_ERROR } from '../translations'; +import { MODEL_GPT_3_5_TURBO } from '../../connectorland/models/model_selector/model_selector'; import { getFormattedMessageContent, getOptionalRequestParams, hasParsableResponse, -} from './helpers'; -import { PerformEvaluationParams } from './settings/evaluation_settings/use_perform_evaluation'; +} from '../helpers'; +import { PerformEvaluationParams } from '../settings/evaluation_settings/use_perform_evaluation'; export interface FetchConnectorExecuteAction { alerts: boolean; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts new file mode 100644 index 0000000000000..fe04a318c8a4a --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; +import { HttpSetup } from '@kbn/core/public'; +import { Conversation } from '../../assistant_context/types'; + +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY = 'elastic_assistant_conversations'; + +export interface BulkActionSummary { + failed: number; + skipped: number; + succeeded: number; + total: number; +} + +export interface BulkActionResult { + updated: Conversation[]; + created: Conversation[]; + deleted: Conversation[]; + skipped: Conversation[]; +} + +export interface BulkActionAggregatedError { + message: string; + status_code: number; + err_code?: string; + rules: Array<{ id: string; name?: string }>; +} + +export interface BulkActionAttributes { + summary: BulkActionSummary; + results: BulkActionResult; + errors?: BulkActionAggregatedError[]; +} + +export interface BulkUpdateResponse { + success?: boolean; + rules_count?: number; + attributes: BulkActionAttributes; +} + +export const bulkConversationsChange = ( + http: HttpSetup, + conversations: { + conversationsToUpdate?: Conversation[]; + conversationsToCreate?: Conversation[]; + conversationsToDelete?: string[]; + } +) => { + return http.fetch(`/api/elastic_assistant/conversations/_bulk_action`, { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify(conversations), + }); +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts new file mode 100644 index 0000000000000..714ad5869cb9b --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart } from '@kbn/core/public'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { useQuery } from '@tanstack/react-query'; +// import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; +import { Conversation } from '../../assistant_context/types'; + +export interface FetchConversationsResponse { + page: number; + perPage: number; + total: number; + data: Conversation[]; +} + +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY = 'elastic_assistant_conversations'; + +const AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; +const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant' as const; +const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations` as const; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = + `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find_user` as const; + +export const useFetchConversationsByUser = () => { + const { http } = useKibana().services; + + const query = { + page: 1, + perPage: 100, + }; + + return useQuery([ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY, query], () => + http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { + method: 'GET', + version: AI_ASSISTANT_API_CURRENT_VERSION, + query, + }) + ); +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 2c35ae9c495e4..eb284a5fdf35b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -83,7 +83,7 @@ export const useChatSend = ({ selectedSystemPrompt: systemPrompt, }); - const updatedMessages = appendMessage({ + const updatedMessages = await appendMessage({ conversationId: currentConversation.id, message, }); @@ -95,7 +95,7 @@ export const useChatSend = ({ const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: updatedMessages, + messages: updatedMessages ?? [], onNewReplacements, replacements: currentConversation.replacements ?? {}, }); @@ -127,12 +127,12 @@ export const useChatSend = ({ replacements: newReplacements, }); - const updatedMessages = removeLastMessage(currentConversation.id); + const updatedMessages = await removeLastMessage(currentConversation.id); const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: updatedMessages, + messages: updatedMessages ?? [], onNewReplacements, replacements: currentConversation.replacements ?? {}, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index a671ad73acabe..13252a8ea892e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -20,7 +20,8 @@ import useEvent from 'react-use/lib/useEvent'; import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; -import { Conversation } from '../../../..'; +import { merge } from 'lodash'; +import { Conversation, useFetchConversationsByUser } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import * as i18n from './translations'; import { DEFAULT_CONVERSATION_TITLE } from '../../use_conversation/translations'; @@ -63,20 +64,34 @@ export const ConversationSelector: React.FC = React.memo( shouldDisableKeyboardShortcut = () => false, isDisabled = false, }) => { - const { allSystemPrompts, conversations } = useAssistantContext(); + const { allSystemPrompts, baseConversations } = useAssistantContext(); - const { deleteConversation, setConversation } = useConversation(); + const { deleteConversation, createConversation } = useConversation(); + + const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + + const conversations = merge( + baseConversations, + (conversationsData?.data ?? []).reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, + {} + ) + ); const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); const conversationOptions = useMemo(() => { return Object.values(conversations).map((conversation) => ({ value: { isDefault: conversation.isDefault ?? false }, - label: conversation.id, + id: conversation.id ?? '', + label: conversation.title, })); }, [conversations]); const [selectedOptions, setSelectedOptions] = useState(() => { - return conversationOptions.filter((c) => c.label === selectedConversationId) ?? []; + return conversationOptions.filter((c) => c.id === selectedConversationId) ?? []; }); // Callback for when user types to create a new system prompt @@ -99,6 +114,7 @@ export const ConversationSelector: React.FC = React.memo( if (!optionExists) { const newConversation: Conversation = { id: searchValue, + title: searchValue, messages: [], apiConfig: { connectorId: defaultConnectorId, @@ -106,7 +122,7 @@ export const ConversationSelector: React.FC = React.memo( defaultSystemPromptId: defaultSystemPrompt?.id, }, }; - setConversation({ conversation: newConversation }); + createConversation(newConversation); } onConversationSelected(searchValue); }, @@ -114,7 +130,7 @@ export const ConversationSelector: React.FC = React.memo( allSystemPrompts, defaultConnectorId, defaultProvider, - setConversation, + createConversation, onConversationSelected, ] ); @@ -187,7 +203,7 @@ export const ConversationSelector: React.FC = React.memo( useEvent('keydown', onKeyDown); useEffect(() => { - setSelectedOptions(conversationOptions.filter((c) => c.label === selectedConversationId)); + setSelectedOptions(conversationOptions.filter((c) => c.id === selectedConversationId)); }, [conversationOptions, selectedConversationId]); const renderOption: ( @@ -195,7 +211,7 @@ export const ConversationSelector: React.FC = React.memo( searchValue: string, OPTION_CONTENT_CLASSNAME: string ) => React.ReactNode = (option, searchValue, contentClassName) => { - const { label, value } = option; + const { label, value, id } = option; return ( = React.memo( color="danger" onClick={(e: React.MouseEvent) => { e.stopPropagation(); - onDelete(label); + onDelete(id ?? ''); }} data-test-subj="delete-option" css={css` diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index 7e79740214a60..e11b2cfede5f1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -19,12 +19,11 @@ import React, { useCallback, useMemo, useState } from 'react'; import { css } from '@emotion/react'; import { Conversation } from '../../../..'; -import { UseAssistantContext } from '../../../assistant_context'; import * as i18n from './translations'; import { SystemPromptSelectorOption } from '../../prompt_editor/system_prompt/system_prompt_modal/system_prompt_selector/system_prompt_selector'; interface Props { - conversations: UseAssistantContext['conversations']; + conversations: Record; onConversationDeleted: (conversationId: string) => void; onConversationSelectionChange: (conversation?: Conversation | string) => void; selectedConversationId?: string; @@ -66,14 +65,15 @@ export const ConversationSelectorSettings: React.FC = React.memo( >(() => { return Object.values(conversations).map((conversation) => ({ value: { isDefault: conversation.isDefault ?? false }, - label: conversation.id, + label: conversation.title, + id: conversation.id ?? '', 'data-test-subj': conversation.id, })); }); const selectedOptions = useMemo(() => { return selectedConversationId - ? conversationOptions.filter((c) => c.label === selectedConversationId) ?? [] + ? conversationOptions.filter((c) => c.id === selectedConversationId) ?? [] : []; }, [conversationOptions, selectedConversationId]); @@ -83,7 +83,8 @@ export const ConversationSelectorSettings: React.FC = React.memo( conversationSelectorSettingsOption.length === 0 ? undefined : Object.values(conversations).find( - (conversation) => conversation.id === conversationSelectorSettingsOption[0]?.label + (conversation) => + conversation.title === conversationSelectorSettingsOption[0]?.label ) ?? conversationSelectorSettingsOption[0]?.label; onConversationSelectionChange(newConversation); }, @@ -131,24 +132,24 @@ export const ConversationSelectorSettings: React.FC = React.memo( // Callback for when user deletes a conversation const onDelete = useCallback( - (label: string) => { - setConversationOptions(conversationOptions.filter((o) => o.label !== label)); - if (selectedOptions?.[0]?.label === label) { + (id: string) => { + setConversationOptions(conversationOptions.filter((o) => o.id !== id)); + if (selectedOptions?.[0]?.id === id) { handleSelectionChange([]); } - onConversationDeleted(label); + onConversationDeleted(id); }, [conversationOptions, handleSelectionChange, onConversationDeleted, selectedOptions] ); const onLeftArrowClick = useCallback(() => { const prevId = getPreviousConversationId(conversationIds, selectedConversationId); - const previousOption = conversationOptions.filter((c) => c.label === prevId); + const previousOption = conversationOptions.filter((c) => c.id === prevId); handleSelectionChange(previousOption); }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); const onRightArrowClick = useCallback(() => { const nextId = getNextConversationId(conversationIds, selectedConversationId); - const nextOption = conversationOptions.filter((c) => c.label === nextId); + const nextOption = conversationOptions.filter((c) => c.id === nextId); handleSelectionChange(nextOption); }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); @@ -157,7 +158,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( searchValue: string, OPTION_CONTENT_CLASSNAME: string ) => React.ReactNode = (option, searchValue, contentClassName) => { - const { label, value } = option; + const { label, value, id } = option; return ( = React.memo( data-test-subj="delete-conversation" onClick={(e: React.MouseEvent) => { e.stopPropagation(); - onDelete(label); + onDelete(id ?? ''); }} css={css` visibility: hidden; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index fa40ff0db7694..d3001f00de603 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -19,7 +19,6 @@ import * as i18nModel from '../../../connectorland/models/model_selector/transla import { ConnectorSelector } from '../../../connectorland/connector_selector'; import { SelectSystemPrompt } from '../../prompt_editor/system_prompt/select_system_prompt'; import { ModelSelector } from '../../../connectorland/models/model_selector/model_selector'; -import { UseAssistantContext } from '../../../assistant_context'; import { ConversationSelectorSettings } from '../conversation_selector_settings'; import { getDefaultSystemPrompt } from '../../use_conversation/helpers'; import { useLoadConnectors } from '../../../connectorland/use_load_connectors'; @@ -27,14 +26,14 @@ import { getGenAiConfig } from '../../../connectorland/helpers'; export interface ConversationSettingsProps { allSystemPrompts: Prompt[]; - conversationSettings: UseAssistantContext['conversations']; + conversationSettings: Record; defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; http: HttpSetup; onSelectedConversationChange: (conversation?: Conversation) => void; selectedConversation: Conversation | undefined; setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction + React.SetStateAction> >; isDisabled?: boolean; } @@ -72,6 +71,7 @@ export const ConversationSettings: React.FC = React.m const newSelectedConversation: Conversation | undefined = isNew ? { id: c ?? '', + title: c ?? '', messages: [], apiConfig: { connectorId: defaultConnectorId, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index acb41a9575581..3eb913b4d8e1c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -41,6 +41,7 @@ const getInitialConversations = (): Record => ({ }, [MOCK_CONVERSATION_TITLE]: { id: MOCK_CONVERSATION_TITLE, + title: MOCK_CONVERSATION_TITLE, messages: [], apiConfig: {}, }, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 86e0f3a460055..14d0d3a1e28e9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -31,6 +31,7 @@ import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { ActionConnectorProps } from '@kbn/triggers-actions-ui-plugin/public/types'; +import { merge } from 'lodash'; import { useChatSend } from './chat_send/use_chat_send'; import { ChatSend } from './chat_send'; import { BlockBotCallToAction } from './block_bot/cta'; @@ -49,6 +50,8 @@ import { QuickPrompts } from './quick_prompts/quick_prompts'; import { useLoadConnectors } from '../connectorland/use_load_connectors'; import { useConnectorSetup } from '../connectorland/connector_setup'; import { ConnectorMissingCallout } from '../connectorland/connector_missing_callout'; +import { useFetchConversationsByUser } from './api/use_fetch_conversations_by_user'; +import { Conversation } from '../assistant_context/types'; export interface Props { conversationId?: string; @@ -75,7 +78,6 @@ const AssistantComponent: React.FC = ({ assistantTelemetry, augmentMessageCodeBlocks, assistantAvailability: { isAssistantEnabled }, - conversations, defaultAllow, defaultAllowReplacement, docLinks, @@ -86,6 +88,7 @@ const AssistantComponent: React.FC = ({ localStorageLastConversationId, title, allSystemPrompts, + baseConversations, } = useAssistantContext(); const [selectedPromptContexts, setSelectedPromptContexts] = useState< @@ -96,8 +99,20 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); - const { amendMessage, createConversation } = useConversation(); + const { amendMessage, getDefaultConversation } = useConversation(); + const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + + const conversations = merge( + baseConversations, + (conversationsData?.data ?? []).reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, + {} + ) + ); // Connector details const { data: connectors, isSuccess: areConnectorsFetched } = useLoadConnectors({ http }); const defaultConnectorId = useMemo(() => getDefaultConnector(connectors)?.id, [connectors]); @@ -130,15 +145,15 @@ const AssistantComponent: React.FC = ({ const currentConversation = useMemo( () => conversations[selectedConversationId] ?? - createConversation({ conversationId: selectedConversationId }), - [conversations, createConversation, selectedConversationId] + getDefaultConversation({ conversationId: selectedConversationId }), + [conversations, getDefaultConversation, selectedConversationId] ); // Welcome setup state const isWelcomeSetup = useMemo(() => { // if any conversation has a connector id, we're not in welcome set up return Object.keys(conversations).some( - (conversation) => conversations[conversation].apiConfig.connectorId != null + (conversation) => conversations[conversation].apiConfig?.connectorId != null ) ? false : (connectors?.length ?? 0) === 0; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx index d53d3cc15e9e2..5e5dfaa5831e0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx @@ -33,7 +33,7 @@ export const ConversationMultiSelector: React.FC = React.memo( const options = useMemo( () => conversations.map((conversation) => ({ - label: conversation.id, + label: conversation.id ?? '', 'data-test-subj': TEST_IDS.CONVERSATIONS_MULTISELECTOR_OPTION(conversation.id), })), [conversations] diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx index 5c02c7d438ec1..3d13388e8891f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx @@ -24,18 +24,17 @@ import { keyBy } from 'lodash/fp'; import { css } from '@emotion/react'; import { Conversation, Prompt } from '../../../../..'; import * as i18n from './translations'; -import { UseAssistantContext } from '../../../../assistant_context'; import { ConversationMultiSelector } from './conversation_multi_selector/conversation_multi_selector'; import { SystemPromptSelector } from './system_prompt_selector/system_prompt_selector'; import { TEST_IDS } from '../../../constants'; interface Props { - conversationSettings: UseAssistantContext['conversations']; + conversationSettings: Record; onSelectedSystemPromptChange: (systemPrompt?: Prompt) => void; selectedSystemPrompt: Prompt | undefined; setUpdatedSystemPromptSettings: React.Dispatch>; setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction + React.SetStateAction> >; systemPromptSettings: Prompt[]; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 0dfd6ebe2904c..c0b337e42dd07 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -6,12 +6,14 @@ */ import React, { useCallback, useState } from 'react'; -import { Prompt, QuickPrompt } from '../../../..'; -import { UseAssistantContext, useAssistantContext } from '../../../assistant_context'; +import { merge } from 'lodash'; +import { Conversation, Prompt, QuickPrompt, useFetchConversationsByUser } from '../../../..'; +import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; +import { bulkConversationsChange } from '../../api/use_bulk_actions_conversations'; interface UseSettingsUpdater { - conversationSettings: UseAssistantContext['conversations']; + conversationSettings: Record; defaultAllow: string[]; defaultAllowReplacement: string[]; knowledgeBase: KnowledgeBaseConfig; @@ -21,11 +23,12 @@ interface UseSettingsUpdater { setUpdatedDefaultAllow: React.Dispatch>; setUpdatedDefaultAllowReplacement: React.Dispatch>; setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction + React.SetStateAction> >; setUpdatedKnowledgeBaseSettings: React.Dispatch>; setUpdatedQuickPromptSettings: React.Dispatch>; setUpdatedSystemPromptSettings: React.Dispatch>; + setDeletedConversationSettings: React.Dispatch>; saveSettings: () => void; } @@ -34,24 +37,38 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { const { allQuickPrompts, allSystemPrompts, - conversations, defaultAllow, defaultAllowReplacement, + baseConversations, knowledgeBase, setAllQuickPrompts, setAllSystemPrompts, - setConversations, setDefaultAllow, setDefaultAllowReplacement, setKnowledgeBase, + http, } = useAssistantContext(); + const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + + const conversations = merge( + baseConversations, + (conversationsData?.data ?? []).reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, + {} + ) + ); + /** * Pending updating state */ // Conversations const [updatedConversationSettings, setUpdatedConversationSettings] = - useState(conversations); + useState>(conversations); + const [deletedConversationSettings, setDeletedConversationSettings] = useState([]); // Quick Prompts const [updatedQuickPromptSettings, setUpdatedQuickPromptSettings] = useState(allQuickPrompts); @@ -71,6 +88,7 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { */ const resetSettings = useCallback((): void => { setUpdatedConversationSettings(conversations); + setDeletedConversationSettings([]); setUpdatedQuickPromptSettings(allQuickPrompts); setUpdatedKnowledgeBaseSettings(knowledgeBase); setUpdatedSystemPromptSettings(allSystemPrompts); @@ -91,14 +109,37 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { const saveSettings = useCallback((): void => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - setConversations(updatedConversationSettings); + + bulkConversationsChange(http, { + conversationsToUpdate: Object.keys(updatedConversationSettings).reduce( + (conversationsToUpdate: Conversation[], conversationId: string) => { + if (!updatedConversationSettings[conversationId].isDefault) { + conversationsToUpdate.push(updatedConversationSettings[conversationId]); + } + return conversationsToUpdate; + }, + [] + ), + conversationsToCreate: Object.keys(updatedConversationSettings).reduce( + (conversationsToCreate: Conversation[], conversationId: string) => { + if (updatedConversationSettings[conversationId].isDefault) { + conversationsToCreate.push(updatedConversationSettings[conversationId]); + } + return conversationsToCreate; + }, + [] + ), + conversationsToDelete: deletedConversationSettings, + }); + setKnowledgeBase(updatedKnowledgeBaseSettings); setDefaultAllow(updatedDefaultAllow); setDefaultAllowReplacement(updatedDefaultAllowReplacement); }, [ + deletedConversationSettings, + http, setAllQuickPrompts, setAllSystemPrompts, - setConversations, setDefaultAllow, setDefaultAllowReplacement, setKnowledgeBase, @@ -125,5 +166,6 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { setUpdatedKnowledgeBaseSettings, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, + setDeletedConversationSettings, }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 3bd9f3fcbff71..c12cb58fb22b6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -7,16 +7,24 @@ import { useCallback } from 'react'; +import { isHttpFetchError } from '@kbn/core-http-browser'; import { useAssistantContext } from '../../assistant_context'; import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; import { ELASTIC_AI_ASSISTANT, ELASTIC_AI_ASSISTANT_TITLE } from './translations'; import { getDefaultSystemPrompt } from './helpers'; +import { + createConversationApi, + deleteConversationApi, + getConversationById, + updateConversationApi, +} from '../api/conversations'; export const DEFAULT_CONVERSATION_STATE: Conversation = { id: i18n.DEFAULT_CONVERSATION_TITLE, messages: [], apiConfig: {}, + title: i18n.DEFAULT_CONVERSATION_TITLE, theme: { title: ELASTIC_AI_ASSISTANT_TITLE, titleIcon: 'logoSecurity', @@ -29,6 +37,7 @@ export const DEFAULT_CONVERSATION_STATE: Conversation = { }, user: {}, }, + user: {}, }; interface AppendMessageProps { @@ -52,186 +61,161 @@ interface CreateConversationProps { interface SetApiConfigProps { conversationId: string; + isDefault?: boolean; + title: string; apiConfig: Conversation['apiConfig']; } -interface SetConversationProps { - conversation: Conversation; -} - interface UseConversation { - appendMessage: ({ conversationId, message }: AppendMessageProps) => Message[]; - amendMessage: ({ conversationId, content }: AmendMessageProps) => void; + appendMessage: ({ + conversationId, + message, + }: AppendMessageProps) => Promise; + amendMessage: ({ conversationId, content }: AmendMessageProps) => Promise; appendReplacements: ({ conversationId, replacements, - }: AppendReplacementsProps) => Record; - clearConversation: (conversationId: string) => void; - createConversation: ({ conversationId, messages }: CreateConversationProps) => Conversation; + }: AppendReplacementsProps) => Promise | undefined>; + clearConversation: (conversationId: string) => Promise; + getDefaultConversation: ({ conversationId, messages }: CreateConversationProps) => Conversation; deleteConversation: (conversationId: string) => void; - removeLastMessage: (conversationId: string) => Message[]; + removeLastMessage: (conversationId: string) => Promise; setApiConfig: ({ conversationId, apiConfig }: SetApiConfigProps) => void; - setConversation: ({ conversation }: SetConversationProps) => void; + createConversation: (conversation: Conversation) => Promise; } export const useConversation = (): UseConversation => { - const { allSystemPrompts, assistantTelemetry, setConversations } = useAssistantContext(); + const { allSystemPrompts, assistantTelemetry, http } = useAssistantContext(); /** * Removes the last message of conversation[] for a given conversationId */ const removeLastMessage = useCallback( - (conversationId: string) => { + async (conversationId: string) => { let messages: Message[] = []; - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - - if (prevConversation != null) { - messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); - const newConversation = { - ...prevConversation, - messages, - }; - return { - ...prev, - [conversationId]: newConversation, - }; - } else { - return prev; - } - }); + const prevConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(prevConversation)) { + return; + } + if (prevConversation != null) { + messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); + await updateConversationApi({ + http, + conversationId, + messages, + }); + } return messages; }, - [setConversations] + [http] ); /** * Updates the last message of conversation[] for a given conversationId with provided content */ const amendMessage = useCallback( - ({ conversationId, content }: AmendMessageProps) => { - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - - if (prevConversation != null) { - const { messages, ...rest } = prevConversation; - const message = messages[messages.length - 1]; - const updatedMessages = message - ? [...messages.slice(0, -1), { ...message, content }] - : [...messages]; - const newConversation = { - ...rest, - messages: updatedMessages, - }; - return { - ...prev, - [conversationId]: newConversation, - }; - } else { - return prev; - } - }); + async ({ conversationId, content }: AmendMessageProps) => { + const prevConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(prevConversation)) { + return; + } + if (prevConversation != null) { + const { messages } = prevConversation; + const message = messages[messages.length - 1]; + const updatedMessages = message + ? [...messages.slice(0, -1), { ...message, content }] + : [...messages]; + await updateConversationApi({ + http, + conversationId, + messages: updatedMessages, + }); + } }, - [setConversations] + [http] ); /** * Append a message to the conversation[] for a given conversationId */ const appendMessage = useCallback( - ({ conversationId, message }: AppendMessageProps): Message[] => { + async ({ conversationId, message }: AppendMessageProps): Promise => { assistantTelemetry?.reportAssistantMessageSent({ conversationId, role: message.role }); let messages: Message[] = []; - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - - if (prevConversation != null) { - messages = [...prevConversation.messages, message]; - const newConversation = { - ...prevConversation, - messages, - }; - return { - ...prev, - [conversationId]: newConversation, - }; - } else { - return prev; - } - }); + const prevConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(prevConversation)) { + return; + } + if (prevConversation != null) { + messages = [...prevConversation.messages, message]; + + await updateConversationApi({ + http, + conversationId, + messages, + }); + } return messages; }, - [assistantTelemetry, setConversations] + [assistantTelemetry, http] ); const appendReplacements = useCallback( - ({ conversationId, replacements }: AppendReplacementsProps): Record => { + async ({ + conversationId, + replacements, + }: AppendReplacementsProps): Promise | undefined> => { let allReplacements = replacements; + const prevConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(prevConversation)) { + return; + } + if (prevConversation != null) { + allReplacements = { + ...prevConversation.replacements, + ...replacements, + }; - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - - if (prevConversation != null) { - allReplacements = { - ...prevConversation.replacements, - ...replacements, - }; - - const newConversation = { - ...prevConversation, - replacements: allReplacements, - }; - - return { - ...prev, - [conversationId]: newConversation, - }; - } else { - return prev; - } - }); + await updateConversationApi({ + http, + conversationId, + replacements: allReplacements, + }); + } return allReplacements; }, - [setConversations] + [http] ); const clearConversation = useCallback( - (conversationId: string) => { - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - const defaultSystemPromptId = getDefaultSystemPrompt({ - allSystemPrompts, - conversation: prevConversation, - })?.id; - - if (prevConversation != null) { - const newConversation: Conversation = { - ...prevConversation, - apiConfig: { - ...prevConversation.apiConfig, - defaultSystemPromptId, - }, - messages: [], - replacements: undefined, - }; + async (conversationId: string) => { + const prevConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(prevConversation)) { + return; + } + const defaultSystemPromptId = getDefaultSystemPrompt({ + allSystemPrompts, + conversation: prevConversation, + })?.id; - return { - ...prev, - [conversationId]: newConversation, - }; - } else { - return prev; - } + await updateConversationApi({ + http, + conversationId, + apiConfig: { + defaultSystemPromptId, + }, + messages: [], + replacements: undefined, }); }, - [allSystemPrompts, setConversations] + [allSystemPrompts, http] ); /** * Create a new conversation with the given conversationId, and optionally add messages */ - const createConversation = useCallback( + const getDefaultConversation = useCallback( ({ conversationId, messages }: CreateConversationProps): Conversation => { const defaultSystemPromptId = getDefaultSystemPrompt({ allSystemPrompts, @@ -247,82 +231,59 @@ export const useConversation = (): UseConversation => { id: conversationId, messages: messages != null ? messages : [], }; - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - if (prevConversation != null) { - throw new Error('Conversation already exists!'); - } else { - return { - ...prev, - [conversationId]: { - ...newConversation, - }, - }; - } - }); return newConversation; }, - [allSystemPrompts, setConversations] + [allSystemPrompts] ); /** - * Delete the conversation with the given conversationId + * Create a new conversation with the given conversation */ - const deleteConversation = useCallback( - (conversationId: string): Conversation | undefined => { - let deletedConversation: Conversation | undefined; - setConversations((prev: Record) => { - const { [conversationId]: prevConversation, ...updatedConversations } = prev; - deletedConversation = prevConversation; - if (prevConversation != null) { - return updatedConversations; - } - return prev; - }); - return deletedConversation; + const createConversation = useCallback( + async (conversation: Conversation): Promise => { + const response = await createConversationApi({ http, conversation }); + if (!isHttpFetchError(response)) { + return response.conversation; + } }, - [setConversations] + [http] ); /** - * Update the apiConfig for a given conversationId + * Delete the conversation with the given conversationId */ - const setApiConfig = useCallback( - ({ conversationId, apiConfig }: SetApiConfigProps): void => { - setConversations((prev: Record) => { - const prevConversation: Conversation | undefined = prev[conversationId]; - - if (prevConversation != null) { - const updatedConversation = { - ...prevConversation, - apiConfig, - }; - - return { - ...prev, - [conversationId]: updatedConversation, - }; - } else { - return prev; - } - }); + const deleteConversation = useCallback( + async (conversationId: string): Promise => { + await deleteConversationApi({ http, id: conversationId }); }, - [setConversations] + [http] ); /** - * Set/overwrite an existing conversation (behaves as createConversation if not already existing) + * Update the apiConfig for a given conversationId */ - const setConversation = useCallback( - ({ conversation }: SetConversationProps): void => { - setConversations((prev: Record) => { - return { - ...prev, - [conversation.id]: conversation, - }; - }); + const setApiConfig = useCallback( + async ({ conversationId, apiConfig, title, isDefault }: SetApiConfigProps): Promise => { + if (isDefault && title === conversationId) { + await createConversationApi({ + http, + conversation: { + apiConfig, + title, + isDefault, + id: '', + messages: [], + }, + }); + } else { + await updateConversationApi({ + http, + conversationId, + apiConfig, + }); + } }, - [setConversations] + [http] ); return { @@ -330,10 +291,10 @@ export const useConversation = (): UseConversation => { appendMessage, appendReplacements, clearConversation, - createConversation, + getDefaultConversation, deleteConversation, removeLastMessage, setApiConfig, - setConversation, + createConversation, }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx index 4fe8af2744263..e88981569a933 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx @@ -15,6 +15,7 @@ import { export const WELCOME_CONVERSATION: Conversation = { id: WELCOME_CONVERSATION_TITLE, + title: WELCOME_CONVERSATION_TITLE, theme: { title: ELASTIC_AI_ASSISTANT_TITLE, titleIcon: 'logoSecurity', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 4d0eec97f2639..d5323ccc97826 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -8,7 +8,7 @@ import { EuiCommentProps } from '@elastic/eui'; import type { HttpSetup } from '@kbn/core-http-browser'; import { omit, uniq } from 'lodash/fp'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import type { IToasts } from '@kbn/core-notifications-browser'; import { ActionTypeRegistryContract } from '@kbn/triggers-actions-ui-plugin/public'; import { useLocalStorage } from 'react-use'; @@ -85,11 +85,11 @@ export interface AssistantProviderProps { showAnonymizedValues: boolean; }) => EuiCommentProps[]; http: HttpSetup; - getInitialConversations: () => Record; + baseConversations: Record; modelEvaluatorEnabled?: boolean; nameSpace?: string; ragOnAlerts?: boolean; - setConversations: React.Dispatch>>; + // setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; title?: string; @@ -114,8 +114,7 @@ export interface UseAssistantContext { basePromptContexts: PromptContextTemplate[]; baseQuickPrompts: QuickPrompt[]; baseSystemPrompts: Prompt[]; - conversationIds: string[]; - conversations: Record; + baseConversations: Record; getComments: ({ currentConversation, showAnonymizedValues, @@ -145,7 +144,7 @@ export interface UseAssistantContext { selectedSettingsTab: SettingsTabs; setAllQuickPrompts: React.Dispatch>; setAllSystemPrompts: React.Dispatch>; - setConversations: React.Dispatch>>; + // setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; setKnowledgeBase: React.Dispatch>; @@ -179,11 +178,10 @@ export const AssistantProvider: React.FC = ({ children, getComments, http, - getInitialConversations, + baseConversations, modelEvaluatorEnabled = false, nameSpace = DEFAULT_ASSISTANT_NAMESPACE, ragOnAlerts = false, - setConversations, setDefaultAllow, setDefaultAllowReplacement, title = DEFAULT_ASSISTANT_TITLE, @@ -261,37 +259,6 @@ export const AssistantProvider: React.FC = ({ */ const [selectedSettingsTab, setSelectedSettingsTab] = useState(CONVERSATIONS_TAB); - const [conversations, setConversationsInternal] = useState(getInitialConversations()); - const conversationIds = useMemo(() => Object.keys(conversations).sort(), [conversations]); - - // TODO: This is a fix for conversations not loading out of localstorage. Also re-introduces our cascading render issue (as it loops back in localstorage) - useEffect(() => { - setConversationsInternal(getInitialConversations()); - }, [getInitialConversations]); - - const onConversationsUpdated = useCallback< - React.Dispatch>> - >( - ( - newConversations: - | Record - | ((prev: Record) => Record) - ) => { - if (typeof newConversations === 'function') { - const updater = newConversations; - setConversationsInternal((prevValue) => { - const newValue = updater(prevValue); - setConversations(newValue); - return newValue; - }); - } else { - setConversations(newConversations); - setConversationsInternal(newConversations); - } - }, - [setConversations] - ); - const value = useMemo( () => ({ actionTypeRegistry, @@ -308,8 +275,6 @@ export const AssistantProvider: React.FC = ({ basePromptContexts, baseQuickPrompts, baseSystemPrompts, - conversationIds, - conversations, defaultAllow: uniq(defaultAllow), defaultAllowReplacement: uniq(defaultAllowReplacement), docLinks, @@ -324,7 +289,7 @@ export const AssistantProvider: React.FC = ({ selectedSettingsTab, setAllQuickPrompts: setLocalStorageQuickPrompts, setAllSystemPrompts: setLocalStorageSystemPrompts, - setConversations: onConversationsUpdated, + // setConversations: onConversationsUpdated, setDefaultAllow, setDefaultAllowReplacement, setKnowledgeBase: setLocalStorageKnowledgeBase, @@ -336,6 +301,7 @@ export const AssistantProvider: React.FC = ({ unRegisterPromptContext, localStorageLastConversationId, setLastConversationId: setLocalStorageLastConversationId, + baseConversations, }), [ actionTypeRegistry, @@ -350,8 +316,6 @@ export const AssistantProvider: React.FC = ({ basePromptContexts, baseQuickPrompts, baseSystemPrompts, - conversationIds, - conversations, defaultAllow, defaultAllowReplacement, docLinks, @@ -363,7 +327,7 @@ export const AssistantProvider: React.FC = ({ localStorageSystemPrompts, modelEvaluatorEnabled, nameSpace, - onConversationsUpdated, + // onConversationsUpdated, promptContexts, ragOnAlerts, registerPromptContext, @@ -378,6 +342,7 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, + baseConversations, ] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 982b74faabf8d..2d497a67cce56 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -50,6 +50,7 @@ export interface ConversationTheme { * */ export interface Conversation { + '@timestamp'?: string; apiConfig: { connectorId?: string; connectorTypeTitle?: string; @@ -57,7 +58,12 @@ export interface Conversation { provider?: OpenAiProviderType; model?: string; }; + user?: { + id?: string; + name?: string; + }; id: string; + title: string; messages: Message[]; replacements?: Record; theme?: ConversationTheme; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx index fa92563883cdb..c7ca37b491738 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx @@ -107,6 +107,8 @@ export const ConnectorSelectorInline: React.FC = React.memo( if (selectedConversation != null) { setApiConfig({ conversationId: selectedConversation.id, + title: selectedConversation.title, + isDefault: selectedConversation.isDefault, apiConfig: { ...selectedConversation.apiConfig, connectorId, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index c80cfe0ed1007..7ce96eccbb415 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -13,14 +13,15 @@ import styled from 'styled-components'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public/common/constants'; import { ActionType } from '@kbn/triggers-actions-ui-plugin/public'; +import { merge } from 'lodash/fp'; import { AddConnectorModal } from '../add_connector_modal'; import { WELCOME_CONVERSATION } from '../../assistant/use_conversation/sample_conversations'; -import { Conversation, Message } from '../../..'; +import { Conversation, Message, useFetchConversationsByUser } from '../../..'; import { useLoadActionTypes } from '../use_load_action_types'; import { StreamingText } from '../../assistant/streaming_text'; import { ConnectorButton } from '../connector_button'; import { useConversation } from '../../assistant/use_conversation'; -import { clearPresentationData, conversationHasNoPresentationData } from './helpers'; +import { conversationHasNoPresentationData } from './helpers'; import * as i18n from '../translations'; import { useAssistantContext } from '../../assistant_context'; import { useLoadConnectors } from '../use_load_connectors'; @@ -47,10 +48,24 @@ export const useConnectorSetup = ({ comments: EuiCommentProps[]; prompt: React.ReactElement; } => { - const { appendMessage, setApiConfig, setConversation } = useConversation(); + const { appendMessage, setApiConfig } = useConversation(); const bottomRef = useRef(null); // Access all conversations so we can add connector to all on initial setup - const { actionTypeRegistry, conversations, http } = useAssistantContext(); + const { actionTypeRegistry, http, baseConversations } = useAssistantContext(); + + const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + + const conversations = merge( + baseConversations, + (conversationsData?.data ?? []).reduce>( + (transformed, conversationData) => { + transformed[conversation.id] = conversationData; + return transformed; + }, + {} + ) + ); + const { data: connectors, isSuccess: areConnectorsFetched, @@ -108,8 +123,8 @@ export const useConnectorSetup = ({ setShowAddConnectorButton(true); bottomRef.current?.scrollIntoView({ block: 'end' }); onSetupComplete?.(); - setConversation({ conversation: clearPresentationData(conversation) }); - }, [conversation, onSetupComplete, setConversation]); + // setConversation({ conversation: clearPresentationData(conversation) }); + }, [onSetupComplete]); // Show button to add connector after last message has finished streaming const handleSkipSetup = useCallback(() => { @@ -185,6 +200,8 @@ export const useConnectorSetup = ({ Object.values(conversations).forEach((c) => { setApiConfig({ conversationId: c.id, + title: c.title, + isDefault: c.isDefault, apiConfig: { ...c.apiConfig, connectorId: connector.id, @@ -194,6 +211,7 @@ export const useConnectorSetup = ({ }, }); }); + console.log('setApiConfig ggg'); refetchConnectors?.(); setIsConnectorModalVisible(false); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts index 5f0c158417a4a..657f95f5a21c1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts @@ -10,6 +10,7 @@ import { Conversation } from '../..'; export const alertConvo: Conversation = { id: 'Alert summary', + title: 'Alert summary', isDefault: true, messages: [ { @@ -32,6 +33,7 @@ export const alertConvo: Conversation = { export const emptyWelcomeConvo: Conversation = { id: 'Welcome', + title: 'Welcome', isDefault: true, theme: { title: 'Elastic AI Assistant', @@ -72,6 +74,7 @@ export const welcomeConvo: Conversation = { export const customConvo: Conversation = { id: 'Custom option', + title: 'Custom option', isDefault: false, messages: [], apiConfig: { diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 775b43952495c..a671d1189754b 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -140,3 +140,5 @@ export type { QuickPrompt } from './impl/assistant/quick_prompts/types'; export type { DeleteKnowledgeBaseResponse } from './impl/assistant/api'; export type { GetKnowledgeBaseStatusResponse } from './impl/assistant/api'; export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; + +export { useFetchConversationsByUser } from './impl/assistant/api/use_fetch_conversations_by_user'; diff --git a/x-pack/plugins/elastic_assistant/common/constants.ts b/x-pack/plugins/elastic_assistant/common/constants.ts index 100aebf395287..13712b169b840 100755 --- a/x-pack/plugins/elastic_assistant/common/constants.ts +++ b/x-pack/plugins/elastic_assistant/common/constants.ts @@ -17,3 +17,17 @@ export const KNOWLEDGE_BASE = `${BASE_PATH}/knowledge_base/{resource?}`; // Model Evaluation export const EVALUATE = `${BASE_PATH}/evaluate`; + +export const MAX_CONVERSATIONS_TO_UPDATE_IN_PARALLEL = 50; +export const CONVERSATIONS_TABLE_MAX_PAGE_SIZE = 100; + +export const ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; +export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_CREATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_create`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_DELETE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_delete`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_UPDATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_update`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find_user`; diff --git a/x-pack/plugins/elastic_assistant/kibana.jsonc b/x-pack/plugins/elastic_assistant/kibana.jsonc index cb89de90fe649..d49531480753a 100644 --- a/x-pack/plugins/elastic_assistant/kibana.jsonc +++ b/x-pack/plugins/elastic_assistant/kibana.jsonc @@ -11,7 +11,9 @@ "actions", "data", "ml", - "taskManager" + "taskManager", + "spaces", + "security" ] } } diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts deleted file mode 100644 index dbdbb1b3b2ce0..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/assistant_data_writer.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; - -interface WriterBulkResponse { - errors: string[]; - docs_written: number; - took: number; -} - -interface BulkParams { - conversations?: string[]; -} - -export interface AssistantDataWriter { - bulk: (params: BulkParams) => Promise; -} - -interface AssistantDataWriterOptions { - esClient: ElasticsearchClient; - index: string; - namespace: string; - logger: Logger; -} - -export class AssistantDataWriter implements AssistantDataWriter { - constructor(private readonly options: AssistantDataWriterOptions) {} - - public bulk = async (params: BulkParams) => { - try { - if (!params.conversations?.length) { - return { errors: [], docs_written: 0, took: 0 }; - } - - const { errors, items, took } = await this.options.esClient.bulk({ - operations: this.buildBulkOperations(params), - }); - - return { - errors: errors - ? items - .map((item) => item.create?.error?.reason) - .filter((error): error is string => !!error) - : [], - docs_written: items.filter( - (item) => item.create?.status === 201 || item.create?.status === 200 - ).length, - took, - }; - } catch (e) { - this.options.logger.error(`Error writing risk scores: ${e.message}`); - return { - errors: [`${e.message}`], - docs_written: 0, - took: 0, - }; - } - }; - - private buildBulkOperations = (params: BulkParams): BulkOperationContainer[] => { - const conversationBody = - params.conversations?.flatMap((conversation) => [ - { create: { _index: this.options.index } }, - // this.scoreToEcs(score, 'host'), - ]) ?? []; - - return conversationBody as BulkOperationContainer[]; - }; - - /* private scoreToEcs = (score: unknown, identifierType: string): unknown => { - const { '@timestamp': _, ...rest } = score; - return { - '@timestamp': score['@timestamp'], - [identifierType]: { - name: score.id_value, - risk: { - ...rest, - }, - }, - }; - };*/ -} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts deleted file mode 100644 index c1fcb2e5ac02e..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_client/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { chunk, flatMap, get, keys } from 'lodash'; -import { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; -import { IIndexPatternString } from '../types'; -import { AssistantDataWriter } from './assistant_data_writer'; -import { getIndexTemplateAndPattern } from '../ai_assistant_service/lib/conversation_configuration_type'; - -// Term queries can take up to 10,000 terms -const CHUNK_SIZE = 10000; - -export interface AIAssistantDataClientParams { - elasticsearchClientPromise: Promise; - kibanaVersion: string; - namespace: string; - logger: Logger; - indexPatternsResorceName: string; -} - -export class AIAssistantDataClient { - private writerCache: Map = new Map(); - - private indexTemplateAndPattern: IIndexPatternString; - - constructor(private readonly options: AIAssistantDataClientParams) { - this.indexTemplateAndPattern = getIndexTemplateAndPattern( - this.options.indexPatternsResorceName, - this.options.namespace ?? DEFAULT_NAMESPACE_STRING - ); - } - - public async getWriter({ namespace }: { namespace: string }): Promise { - if (this.writerCache.get(namespace)) { - return this.writerCache.get(namespace) as AssistantDataWriter; - } - const indexPatterns = this.indexTemplateAndPattern; - await this.initializeWriter(namespace, indexPatterns.alias); - return this.writerCache.get(namespace) as AssistantDataWriter; - } - - private async initializeWriter(namespace: string, index: string): Promise { - const esClient = await this.options.elasticsearchClientPromise; - const writer = new AssistantDataWriter({ - esClient, - namespace, - index, - logger: this.options.logger, - }); - - this.writerCache.set(namespace, writer); - return writer; - } -} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 185d0ae24c2b4..4100c669a0765 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -14,17 +14,17 @@ import { import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; -import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; +// import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; import { AssistantResourceNames } from '../types'; import { conversationsFieldMap, getIndexTemplateAndPattern, - mappingComponentName, totalFieldsLimit, } from './lib/conversation_configuration_type'; import { createConcreteWriteIndex } from './lib/create_concrete_write_index'; import { DataStreamAdapter } from './lib/create_datastream'; -import { AIAssistantDataClient } from '../ai_assistant_data_client'; +import { AIAssistantDataClient } from '../conversations_data_client'; import { InitializationPromise, ResourceInstallationHelper, @@ -32,7 +32,7 @@ import { errorResult, successResult, } from './create_resource_installation_helper'; -import { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; +// import { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; export const ECS_CONTEXT = `ecs`; @@ -40,7 +40,7 @@ function getResourceName(resource: string) { return `.kibana-elastic-ai-assistant-${resource}`; } -export const getComponentTemplateName = (name: string) => `.alerts-${name}-mappings`; +// export const getComponentTemplateName = (name: string) => `.alerts-${name}-mappings`; interface AIAssistantServiceOpts { logger: Logger; @@ -53,6 +53,7 @@ interface AIAssistantServiceOpts { export interface CreateAIAssistantClientParams { logger: Logger; namespace: string; + currentUser: AuthenticatedUser | null; } export class AIAssistantService { @@ -155,7 +156,8 @@ export class AIAssistantService { elasticsearchClientPromise: this.options.elasticsearchClientPromise, namespace: opts.namespace, kibanaVersion: this.options.kibanaVersion, - indexPatternsResorceName: this.resourceNames.indexPatterns.conversations, + indexPatternsResorceName: 'kibana-elastic-ai-assistant-conversations', + currentUser: opts.currentUser, }); } @@ -194,9 +196,8 @@ export class AIAssistantService { this.options.logger.debug(`Initializing resources for AIAssistantService`); const esClient = await this.options.elasticsearchClientPromise; - // TODO: add DLM policy await Promise.all([ - createOrUpdateComponentTemplate({ + /* createOrUpdateComponentTemplate({ logger: this.options.logger, esClient, template: getComponentTemplateFromFieldMap({ @@ -206,12 +207,13 @@ export class AIAssistantService { includeSettings: true, }), totalFieldsLimit, - }), + }), */ createOrUpdateComponentTemplate({ logger: this.options.logger, esClient, template: { name: this.resourceNames.componentTemplate.conversations, + // TODO: add DLM policy _meta: { managed: true, }, @@ -245,14 +247,32 @@ export class AIAssistantService { version: this.options.kibanaVersion, }, managed: true, - namespace, + namespace: namespace ?? 'default', }; const indexPatterns = getIndexTemplateAndPattern( - this.resourceNames.indexPatterns.conversations, + 'kibana-elastic-ai-assistant-conversations', namespace ?? 'default' ); + /* + export const getIndexTemplateAndPattern = ( + context: string, + namespace?: string +): IIndexPatternString => { + const concreteNamespace = namespace ? namespace : DEFAULT_NAMESPACE_STRING; + const pattern = `${context}`; + const patternWithNamespace = `${pattern}-${concreteNamespace}`; + return { + template: `${patternWithNamespace}-index-template`, + pattern: `.internal.${patternWithNamespace}-*`, + basePattern: `.${pattern}-*`, + name: `.internal.${patternWithNamespace}-000001`, + alias: `.${patternWithNamespace}`, + }; +}; +*/ + await createOrUpdateIndexTemplate({ logger: this.options.logger, esClient, @@ -260,11 +280,13 @@ export class AIAssistantService { name: indexPatterns.template, body: { data_stream: { hidden: true }, - index_patterns: [indexPatterns.alias], - composed_of: [mappingComponentName], + index_patterns: indexPatterns.pattern, + composed_of: [this.resourceNames.componentTemplate.conversations], template: { lifecycle: {}, settings: { + hidden: true, + 'index.mapping.ignore_malformed': true, 'index.mapping.total_fields.limit': totalFieldsLimit, }, mappings: { diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts index c64b98d3f7c9e..c178549c542b4 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts @@ -5,7 +5,6 @@ * 2.0. */ import type { FieldMap } from '@kbn/alerts-as-data-utils'; -import { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import { IIndexPatternString } from '../../types'; @@ -27,78 +26,6 @@ const dynamic = { dynamic: true, }; -const commonFields: ClusterComponentTemplate['component_template']['template'] = { - mappings: { - dynamic_templates: [ - { - numeric_labels: { - path_match: 'numeric_labels.*', - mapping: { - scaling_factor: 1000000, - type: 'scaled_float', - }, - }, - }, - ], - dynamic: false, - properties: { - '@timestamp': date, - labels: dynamic, - numeric_labels: dynamic, - user: { - properties: { - id: keyword, - name: keyword, - }, - }, - conversation: { - properties: { - id: keyword, - title: text, - last_updated: date, - }, - }, - api_config: { - properties: { - connectorId: keyword, - connectorTypeTitle: text, - model: keyword, - provider: keyword, - }, - }, - anonymized_fields: { - type: 'object', - properties: { - field_name: keyword, - value: { - type: 'object', - enabled: false, - }, - uuid: keyword, - }, - }, - namespace: keyword, - messages: { - type: 'object', - properties: { - '@timestamp': date, - message: { - type: 'object', - properties: { - content: text, - event: text, - role: keyword, - }, - }, - }, - }, - public: { - type: 'boolean', - }, - }, - }, -}; - export const conversationsFieldMap: FieldMap = { '@timestamp': { type: 'date', @@ -115,18 +42,28 @@ export const conversationsFieldMap: FieldMap = { array: false, required: false, }, - 'conversation.id': { + id: { type: 'keyword', array: false, + required: true, + }, + title: { + type: 'keyword', + array: false, + required: true, + }, + is_default: { + type: 'boolean', + array: false, required: false, }, - 'conversation.title': { - type: 'object', + updated_at: { + type: 'date', array: false, required: false, }, - 'conversation.last_updated': { - type: 'object', + created_at: { + type: 'date', array: false, required: false, }, @@ -135,7 +72,7 @@ export const conversationsFieldMap: FieldMap = { array: true, required: false, }, - 'messages.id': { + 'messages.@timestamp': { type: 'keyword', array: false, required: true, @@ -145,19 +82,84 @@ export const conversationsFieldMap: FieldMap = { array: false, required: true, }, - 'messages.event': { - type: 'keyword', + 'messages.is_error': { + type: 'boolean', array: false, - required: true, + required: false, }, 'messages.content': { type: 'keyword', array: false, required: false, }, - 'messages.anonymized_fields': { + 'messages.reader': { + type: 'keyword', + array: false, + required: false, + }, + 'messages.replacements': { type: 'object', - array: true, + array: false, + required: false, + }, + 'messages.presentation': { + type: 'object', + array: false, + required: false, + }, + 'messages.presentation.delay': { + type: 'long', + array: false, + required: false, + }, + 'messages.presentation.stream': { + type: 'boolean', + array: false, + required: false, + }, + 'messages.trace_data': { + type: 'object', + array: false, + required: false, + }, + 'messages.trace_data.transaction_id': { + type: 'keyword', + array: false, + required: false, + }, + 'messages.trace_data.trace_id': { + type: 'keyword', + array: false, + required: false, + }, + api_config: { + type: 'object', + array: false, + required: false, + }, + 'api_config.connector_id': { + type: 'keyword', + array: false, + required: false, + }, + 'api_config.connector_type_title': { + type: 'keyword', + array: false, + required: false, + }, + 'api_config.default_system_prompt_id': { + type: 'keyword', + array: false, + required: false, + }, + 'api_config.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'api_config.model': { + type: 'keyword', + array: false, required: false, }, } as const; @@ -173,10 +175,10 @@ export const getIndexTemplateAndPattern = ( const pattern = `${context}`; const patternWithNamespace = `${pattern}-${concreteNamespace}`; return { - template: `${patternWithNamespace}-index-template`, - pattern: `.internal.${patternWithNamespace}-*`, + template: `.${patternWithNamespace}-index-template`, + pattern: `.${patternWithNamespace}*`, basePattern: `.${pattern}-*`, - name: `.internal.${patternWithNamespace}-000001`, + name: `.${patternWithNamespace}-000001`, alias: `.${patternWithNamespace}`, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts index dd14a969f644c..fa87000163f41 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts @@ -9,8 +9,8 @@ import { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib import { Logger, ElasticsearchClient } from '@kbn/core/server'; import { get } from 'lodash'; import { retryTransientEsErrors } from './retry_transient_es_errors'; -import { DataStreamAdapter } from './create_data_stream'; -import { IIndexPatternString } from '../types'; +import { DataStreamAdapter } from './create_datastream'; +import { IIndexPatternString } from '../../types'; export interface ConcreteIndexInfo { index: string; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts new file mode 100644 index 0000000000000..be783cf0dd61d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { + ConversationCreateProps, + ConversationUpdateProps, + UUID, +} from '../schemas/conversations/common_attributes.gen'; +import { transformToCreateScheme } from './create_conversation'; +import { transformToUpdateScheme } from './update_conversation'; + +interface WriterBulkResponse { + errors: string[]; + docs_created: string[]; + docs_deleted: string[]; + docs_updated: string[]; + took: number; +} + +interface BulkParams { + conversationsToCreate?: ConversationCreateProps[]; + conversationsToUpdate?: ConversationUpdateProps[]; + conversationsToDelete?: string[]; +} + +export interface ConversationDataWriter { + bulk: (params: BulkParams) => Promise; +} + +interface ConversationDataWriterOptions { + esClient: ElasticsearchClient; + index: string; + namespace: string; + user: { id?: UUID; name?: string }; + logger: Logger; +} + +export class ConversationDataWriter implements ConversationDataWriter { + constructor(private readonly options: ConversationDataWriterOptions) {} + + public bulk = async (params: BulkParams) => { + try { + if ( + !params.conversationsToCreate?.length && + !params.conversationsToUpdate?.length && + !params.conversationsToDelete?.length + ) { + return { errors: [], docs_created: [], docs_deleted: [], docs_updated: [], took: 0 }; + } + + const { errors, items, took } = await this.options.esClient.bulk({ + operations: this.buildBulkOperations(params), + }); + + return { + errors: errors + ? items + .map((item) => item.create?.error?.reason) + .filter((error): error is string => !!error) + : [], + docs_created: items + .filter((item) => item.create?.status === 201 || item.create?.status === 200) + .map((item) => item.create?._id ?? ''), + docs_deleted: items + .filter((item) => item.delete?.status === 201 || item.delete?.status === 200) + .map((item) => item.delete?._id ?? ''), + docs_updated: items + .filter((item) => item.update?.status === 201 || item.update?.status === 200) + .map((item) => item.update?._id ?? ''), + took, + }; + } catch (e) { + this.options.logger.error(`Error bulk actions for conversations: ${e.message}`); + return { + errors: [`${e.message}`], + docs_created: [], + docs_deleted: [], + docs_updated: [], + took: 0, + }; + } + }; + + private buildBulkOperations = (params: BulkParams): BulkOperationContainer[] => { + const changedAt = new Date().toISOString(); + const conversationBody = + params.conversationsToCreate?.flatMap((conversation) => [ + { create: { _index: this.options.index } }, + transformToCreateScheme(changedAt, this.options.namespace, this.options.user, conversation), + ]) ?? []; + + const conversationUpdatedBody = + params.conversationsToUpdate?.flatMap((conversation) => [ + { create: { _id: conversation.id, _index: this.options.index } }, + transformToUpdateScheme(changedAt, conversation), + ]) ?? []; + + const conversationDeletedBody = + params.conversationsToDelete?.flatMap((conversationId) => [ + { create: { _id: conversationId, _index: this.options.index } }, + ]) ?? []; + + return [ + ...conversationBody, + ...conversationUpdatedBody, + ...conversationDeletedBody, + ] as BulkOperationContainer[]; + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts new file mode 100644 index 0000000000000..c5272a02edca0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import { + ConversationCreateProps, + ConversationResponse, + Replacement, + UUID, +} from '../schemas/conversations/common_attributes.gen'; + +export interface CreateMessageSchema { + '@timestamp'?: string; + created_at: string; + title: string; + id?: string | undefined; + messages?: Array<{ + '@timestamp': string; + content: string; + reader?: string | undefined; + replacements?: unknown; + role: 'user' | 'assistant' | 'system'; + is_error?: boolean; + presentation?: { + delay?: number; + stream?: boolean; + }; + trace_data?: { + transaction_id?: string; + trace_id?: string; + }; + }>; + api_config?: { + connector_id?: string; + connector_type_title?: string; + default_system_prompt_id?: string; + provider?: 'OpenAI' | 'Azure OpenAI'; + model?: string; + }; + is_default?: boolean; + exclude_from_last_conversation_storage?: boolean; + replacements?: unknown; + user?: { + id?: string; + name?: string; + }; + updated_at?: string; + namespace: string; +} + +export const createConversation = async ( + esClient: ElasticsearchClient, + conversationIndex: string, + namespace: string, + user: { id?: UUID; name?: string }, + conversation: ConversationCreateProps +): Promise => { + const createdAt = new Date().toISOString(); + const body: CreateMessageSchema = transformToCreateScheme( + createdAt, + namespace, + user, + conversation + ); + + const response = await esClient.create({ + body, + id: conversation.isDefault && conversation.id ? conversation.id : uuidv4(), + index: conversationIndex, + refresh: 'wait_for', + }); + + return { + id: response._id, + ...transform(body), + }; +}; + +export const transformToCreateScheme = ( + createdAt: string, + namespace: string, + user: { id?: UUID; name?: string }, + { + id, + title, + apiConfig, + excludeFromLastConversationStorage, + isDefault, + messages, + replacements, + }: ConversationCreateProps +) => { + return { + '@timestamp': createdAt, + created_at: createdAt, + user, + title, + api_config: { + connector_id: apiConfig?.connectorId, + connector_type_title: apiConfig?.connectorTypeTitle, + default_system_prompt_id: apiConfig?.defaultSystemPromptId, + model: apiConfig?.model, + provider: apiConfig?.provider, + }, + exclude_from_last_conversation_storage: excludeFromLastConversationStorage, + is_default: isDefault, + messages: messages?.map((message) => ({ + '@timestamp': message.timestamp, + content: message.content, + is_error: message.isError, + presentation: message.presentation, + reader: message.reader, + replacements: message.replacements, + role: message.role, + trace_data: { + trace_id: message.traceData?.traceId, + transaction_id: message.traceData?.transactionId, + }, + })), + updated_at: createdAt, + replacements, + namespace, + }; +}; + +function transform(conversationSchema: CreateMessageSchema): ConversationResponse { + const response: ConversationResponse = { + timestamp: conversationSchema['@timestamp'], + createdAt: conversationSchema.created_at, + user: conversationSchema.user, + title: conversationSchema.title, + apiConfig: { + connectorId: conversationSchema.api_config?.connector_id, + connectorTypeTitle: conversationSchema.api_config?.connector_type_title, + defaultSystemPromptId: conversationSchema.api_config?.default_system_prompt_id, + model: conversationSchema.api_config?.model, + provider: conversationSchema.api_config?.provider, + }, + excludeFromLastConversationStorage: conversationSchema.exclude_from_last_conversation_storage, + isDefault: conversationSchema.is_default, + messages: conversationSchema.messages?.map((message) => ({ + timestamp: message['@timestamp'], + content: message.content, + isError: message.is_error, + presentation: message.presentation, + reader: message.reader, + replacements: message.replacements as Replacement[], + role: message.role, + traceData: { + traceId: message.trace_data?.trace_id, + transactionId: message.trace_data?.transaction_id, + }, + })), + updatedAt: conversationSchema.updated_at, + replacements: conversationSchema.replacements as Replacement[], + namespace: conversationSchema.namespace, + }; + return response; +} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts new file mode 100644 index 0000000000000..8f5aae3cc6af8 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { waitUntilDocumentIndexed } from '../lib/wait_until_document_indexed'; +import { getConversation } from './get_conversation'; + +export const deleteConversation = async ( + esClient: ElasticsearchClient, + conversationIndex: string, + id: string +): Promise => { + const conversation = await getConversation(esClient, conversationIndex, id); + if (conversation !== null) { + const response = await esClient.deleteByQuery({ + body: { + query: { + ids: { + values: [id], + }, + }, + }, + conflicts: 'proceed', + index: conversationIndex, + refresh: false, + }); + + if (response.deleted) { + const checkIfConversationDeleted = async (): Promise => { + const deletedConversation = await getConversation(esClient, conversationIndex, id); + if (deletedConversation !== null) { + throw Error('Conversation has not been re-indexed in time'); + } + }; + + await waitUntilDocumentIndexed(checkIfConversationDeleted); + } else { + throw Error('No conversation has been deleted'); + } + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts new file mode 100644 index 0000000000000..5d242c09451a2 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingRuntimeFields, Sort } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import { estypes } from '@elastic/elasticsearch'; +import { EsQueryConfig, Query, buildEsQuery } from '@kbn/es-query'; +import { transformESToConversations } from './transforms'; +import { SearchEsConversationSchema } from './types'; +import { FindConversationsResponse } from '../schemas/conversations/find_conversations_route.gen'; + +interface FindConversationsOptions { + filter?: string; + fields?: string[]; + perPage: number; + page: number; + sortField?: string; + sortOrder?: estypes.SortOrder; + esClient: ElasticsearchClient; + conversationIndex: string; + runtimeMappings?: MappingRuntimeFields | undefined; +} + +export const findConversations = async ({ + esClient, + filter, + page, + perPage, + sortField, + conversationIndex, + fields, + sortOrder, +}: FindConversationsOptions): Promise => { + const query = getQueryFilter({ filter }); + let sort: Sort | undefined; + const ascOrDesc = sortOrder ?? ('asc' as const); + if (sortField != null) { + sort = [{ [sortField]: ascOrDesc }]; + } + const response = await esClient.search({ + body: { + query, + track_total_hits: true, + sort, + }, + _source: false, + from: (page - 1) * perPage, + ignore_unavailable: true, + index: conversationIndex, + seq_no_primary_term: true, + size: perPage, + fields: fields ?? ['*'], + }); + return { + data: transformESToConversations(response), + page, + perPage, + total: + (typeof response.hits.total === 'number' + ? response.hits.total // This format is to be removed in 8.0 + : response.hits.total?.value) ?? 0, + }; +}; + +export interface GetQueryFilterOptions { + filter?: string; +} + +export const getQueryFilter = ({ filter }: GetQueryFilterOptions) => { + const kqlQuery: Query | Query[] = filter + ? { + language: 'kuery', + query: filter, + } + : []; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + dateFormatTZ: 'Zulu', + ignoreFilterIfFieldNotInIndex: false, + queryStringOptions: { analyze_wildcard: true }, + }; + + return buildEsQuery(undefined, kqlQuery, [], config); +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts new file mode 100644 index 0000000000000..af995d5b41f62 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; +import { SearchEsConversationSchema } from './types'; +import { transformESToConversations } from './transforms'; + +export const getConversation = async ( + esClient: ElasticsearchClient, + conversationIndex: string, + id: string +): Promise => { + const response = await esClient.search({ + body: { + query: { + term: { + _id: id, + }, + }, + }, + _source: false, + fields: ['*'], + ignore_unavailable: true, + index: conversationIndex, + seq_no_primary_term: true, + }); + const conversation = transformESToConversations(response); + return conversation[0] ?? null; +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts new file mode 100644 index 0000000000000..4e9abe8252b64 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; +import { estypes } from '@elastic/elasticsearch'; +import { IIndexPatternString } from '../types'; +import { ConversationDataWriter } from './conversations_data_writer'; +import { getIndexTemplateAndPattern } from '../ai_assistant_service/lib/conversation_configuration_type'; +import { createConversation } from './create_conversation'; +import { + ConversationCreateProps, + ConversationResponse, + ConversationUpdateProps, +} from '../schemas/conversations/common_attributes.gen'; +import { FindConversationsResponse } from '../schemas/conversations/find_conversations_route.gen'; +import { findConversations } from './find_conversations'; +import { updateConversation } from './update_conversation'; +import { getConversation } from './get_conversation'; +import { deleteConversation } from './delete_conversation'; + +export enum OpenAiProviderType { + OpenAi = 'OpenAI', + AzureAi = 'Azure OpenAI', +} + +export interface AIAssistantDataClientParams { + elasticsearchClientPromise: Promise; + kibanaVersion: string; + namespace: string; + logger: Logger; + indexPatternsResorceName: string; + currentUser: AuthenticatedUser | null; +} + +export class AIAssistantDataClient { + /** Kibana space id the conversation are part of */ + private readonly spaceId: string; + + /** User creating, modifying, deleting, or updating a conversation */ + private readonly currentUser: AuthenticatedUser | null; + + private writerCache: Map = new Map(); + + private indexTemplateAndPattern: IIndexPatternString; + + constructor(private readonly options: AIAssistantDataClientParams) { + this.indexTemplateAndPattern = getIndexTemplateAndPattern( + this.options.indexPatternsResorceName, + this.options.namespace ?? DEFAULT_NAMESPACE_STRING + ); + this.currentUser = this.options.currentUser; + this.spaceId = this.options.namespace; + } + + public async getWriter(): Promise { + const namespace = this.spaceId; + if (this.writerCache.get(namespace)) { + return this.writerCache.get(namespace) as ConversationDataWriter; + } + const indexPatterns = this.indexTemplateAndPattern; + await this.initializeWriter(namespace, indexPatterns.alias); + return this.writerCache.get(namespace) as ConversationDataWriter; + } + + private async initializeWriter( + namespace: string, + index: string + ): Promise { + const esClient = await this.options.elasticsearchClientPromise; + const writer = new ConversationDataWriter({ + esClient, + namespace, + index, + logger: this.options.logger, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + }); + + this.writerCache.set(namespace, writer); + return writer; + } + + public getReader(options: { namespace?: string } = {}) { + const indexPatterns = this.indexTemplateAndPattern.alias; + + return { + search: async < + TSearchRequest extends ESSearchRequest, + TConversationDoc = Partial + >( + request: TSearchRequest + ): Promise> => { + try { + const esClient = await this.options.elasticsearchClientPromise; + return (await esClient.search({ + ...request, + index: indexPatterns, + ignore_unavailable: true, + seq_no_primary_term: true, + })) as unknown as ESSearchResponse; + } catch (err) { + this.options.logger.error( + `Error performing search in AIAssistantDataClient - ${err.message}` + ); + throw err; + } + }, + }; + } + + public getConversation = async (id: string): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + return getConversation(esClient, this.indexTemplateAndPattern.alias, id); + }; + + public findConversations = async ({ + perPage, + page, + sortField, + sortOrder, + filter, + fields, + }: { + perPage: number; + page: number; + sortField?: string; + sortOrder?: string; + filter?: string; + fields?: string[]; + }): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + return findConversations({ + esClient, + fields, + page, + perPage, + filter, + sortField, + conversationIndex: this.indexTemplateAndPattern.alias, + sortOrder: sortOrder as estypes.SortOrder, + }); + }; + + /** + * Creates a conversation, if given at least the "title" and "apiConfig" + * See {@link https://www.elastic.co/guide/en/security/current/} + * for more information around formats of the deserializer and serializer + * @param options + * @param options.id The id of the conversat to create or "undefined" if you want an "id" to be auto-created for you + * @param options.title A custom deserializer for the conversation. Optionally, you an define this as handle bars. See online docs for more information. + * @param options.messages Set this to true if this is a conversation that is "immutable"/"pre-packaged". + * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. + * @returns The conversation created + */ + public createConversation = async ( + props: ConversationCreateProps + ): Promise => { + const { currentUser } = this; + const esClient = await this.options.elasticsearchClientPromise; + return createConversation( + esClient, + this.indexTemplateAndPattern.alias, + this.spaceId, + { id: currentUser?.profile_uid, name: currentUser?.username }, + props + ); + }; + + /** + * Updates a conversation container's value given the id of the conversation. + * See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html} + * for more information around optimistic concurrency control. + * @param options + * @param options._version This is the version, useful for optimistic concurrency control. + * @param options.id id of the conversation to replace the conversation container data with. + * @param options.name The new name, or "undefined" if this should not be updated. + * @param options.description The new description, or "undefined" if this should not be updated. + * @param options.meta Additional meta data to associate with the conversation items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. + * @param options.version Updates the version of the conversation. + */ + public updateConversation = async ( + existingConversation: ConversationResponse, + updatedProps: ConversationUpdateProps, + isPatch?: boolean + ): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + return updateConversation( + esClient, + this.indexTemplateAndPattern.alias, + existingConversation, + updatedProps, + isPatch + ); + }; + + /** + * Given a conversation id, this will delete the conversation from the id + * @param options + * @param options.id The id of the conversation to delete + * @returns The conversation deleted if found, otherwise null + */ + public deleteConversation = async (id: string): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + deleteConversation(esClient, this.indexTemplateAndPattern.alias, id); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts new file mode 100644 index 0000000000000..1dfadfd81f425 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { SearchEsConversationSchema } from './types'; +import { ConversationResponse, Replacement } from '../schemas/conversations/common_attributes.gen'; + +export const transformESToConversations = ( + response: estypes.SearchResponse +): ConversationResponse[] => { + return response.hits.hits.map((hit) => { + const conversationSchema = hit.fields; + const conversation: ConversationResponse = { + timestamp: conversationSchema?.['@timestamp']?.[0], + createdAt: conversationSchema?.created_at?.[0], + user: { + id: conversationSchema?.['user.id']?.[0], + name: conversationSchema?.['user.name']?.[0], + }, + title: conversationSchema?.title?.[0], + apiConfig: { + connectorId: conversationSchema?.['api_config.connector_id']?.[0], + connectorTypeTitle: conversationSchema?.['api_config.connector_type_title']?.[0], + defaultSystemPromptId: conversationSchema?.['api_config.default_system_prompt_id']?.[0], + model: conversationSchema?.['api_config.model']?.[0], + provider: conversationSchema?.['api_config.provider']?.[0], + }, + excludeFromLastConversationStorage: + conversationSchema?.exclude_from_last_conversation_storage?.[0], + isDefault: conversationSchema?.is_default?.[0], + messages: + // eslint-disable-next-line @typescript-eslint/no-explicit-any + conversationSchema?.messages?.map((message: Record) => ({ + timestamp: message['@timestamp'], + content: message.content, + isError: message.is_error, + presentation: message.presentation, + reader: message.reader, + replacements: message.replacements as Replacement[], + role: message.role, + traceData: { + traceId: message?.['trace_data.trace_id'], + transactionId: message?.['trace_data.transaction_id'], + }, + })) ?? [], + updatedAt: conversationSchema?.updated_at?.[0], + replacements: conversationSchema?.replacements?.[0] as Replacement[], + namespace: conversationSchema?.namespace?.[0], + id: hit._id, + }; + + return conversation; + }); +}; + +export const encodeHitVersion = (hit: T): string | undefined => { + // Have to do this "as cast" here as these two types aren't included in the SearchResponse hit type + const { _seq_no: seqNo, _primary_term: primaryTerm } = hit as unknown as { + _seq_no: number; + _primary_term: number; + }; + + if (seqNo == null || primaryTerm == null) { + return undefined; + } else { + return Buffer.from(JSON.stringify([seqNo, primaryTerm]), 'utf8').toString('base64'); + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts new file mode 100644 index 0000000000000..999dd03d3e5fa --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface SearchEsConversationSchema { + id: string; + '@timestamp': string; + created_at: string; + title: string; + messages?: Array<{ + '@timestamp': string; + content: string; + reader?: string | undefined; + replacements?: unknown; + role: 'user' | 'assistant' | 'system'; + is_error?: boolean; + presentation?: { + delay?: number; + stream?: boolean; + }; + trace_data?: { + transaction_id?: string; + trace_id?: string; + }; + }>; + api_config?: { + connector_id?: string; + connector_type_title?: string; + default_system_prompt_id?: string; + provider?: 'OpenAI' | 'Azure OpenAI'; + model?: string; + }; + is_default?: boolean; + exclude_from_last_conversation_storage?: boolean; + replacements?: unknown; + user?: { + id?: string; + name?: string; + }; + updated_at?: string; + namespace: string; +} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts new file mode 100644 index 0000000000000..6bfbd3b4cbe33 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { + ConversationResponse, + ConversationUpdateProps, +} from '../schemas/conversations/common_attributes.gen'; +import { waitUntilDocumentIndexed } from '../lib/wait_until_document_indexed'; +import { getConversation } from './get_conversation'; + +export interface UpdateConversationSchema { + '@timestamp'?: string; + title?: string; + messages?: Array<{ + '@timestamp': string; + content: string; + reader?: string | undefined; + replacements?: unknown; + role: 'user' | 'assistant' | 'system'; + is_error?: boolean; + presentation?: { + delay?: number; + stream?: boolean; + }; + trace_data?: { + transaction_id?: string; + trace_id?: string; + }; + }>; + api_config?: { + connector_id?: string; + connector_type_title?: string; + default_system_prompt_id?: string; + provider?: 'OpenAI' | 'Azure OpenAI'; + model?: string; + }; + exclude_from_last_conversation_storage?: boolean; + replacements?: unknown; + updated_at?: string; +} + +export const updateConversation = async ( + esClient: ElasticsearchClient, + conversationIndex: string, + existingConversation: ConversationResponse, + conversation: ConversationUpdateProps, + isPatch?: boolean +): Promise => { + const updatedAt = new Date().toISOString(); + + // checkVersionConflict(_version, list._version); + // const calculatedVersion = version == null ? list.version + 1 : version; + + const params: UpdateConversationSchema = transformToUpdateScheme(updatedAt, conversation); + + const response = await esClient.updateByQuery({ + conflicts: 'proceed', + index: conversationIndex, + query: { + ids: { + values: [existingConversation.id ?? ''], + }, + }, + refresh: false, + script: { + lang: 'painless', + params: { + ...params, + // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !isPatch, + }, + source: ` + if (params.assignEmpty == true || params.containsKey('api_config')) { + if (params.assignEmpty == true || params.containsKey('api_config.connector_id')) { + ctx._source.api_config.connector_id = params.api_config.connector_id; + } + if (params.assignEmpty == true || params.containsKey('api_config.connector_type_title')) { + ctx._source.api_config.connector_type_title = params.api_config.connector_type_title; + } + if (params.assignEmpty == true || params.containsKey('api_config.default_system_prompt_id')) { + ctx._source.api_config.default_system_prompt_id = params.api_config.default_system_prompt_id; + } + if (params.assignEmpty == true || params.containsKey('api_config.model')) { + ctx._source.api_config.model = params.api_config.model; + } + if (params.assignEmpty == true || params.containsKey('api_config.provider')) { + ctx._source.api_config.provider = params.api_config.provider; + } + } + if (params.assignEmpty == true || params.containsKey('exclude_from_last_conversation_storage')) { + ctx._source.exclude_from_last_conversation_storage = params.exclude_from_last_conversation_storage; + } + if (params.assignEmpty == true || params.containsKey('replacements')) { + ctx._source.replacements = params.replacements; + } + if (params.assignEmpty == true || params.containsKey('title')) { + ctx._source.title = params.title; + } + ctx._source.updated_at = params.updated_at; + if (params.assignEmpty == true || params.containsKey('messages')) { + ctx._source.messages = []; + for (message in params.messages) { + def newMessage = [:]; + newMessage['@timestamp'] = message['@timestamp']; + newMessage.content = message.content; + newMessage.is_error = message.is_error; + newMessage.presentation = message.presentation; + newMessage.reader = message.reader; + newMessage.replacements = message.replacements; + newMessage.role = message.role; + newMessage.trace_data.trace_id = message.trace_data.trace_id; + newMessage.trace_data.transaction_id = message.trace_data.transaction_id; + ctx._source.messages.add(enrichment); + } + } + `, + }, + }); + + let updatedOCCVersion: string | undefined; + if (response.updated) { + const checkIfListUpdated = async (): Promise => { + const updatedConversation = await getConversation( + esClient, + conversationIndex, + existingConversation.id ?? '' + ); + /* if (updatedList?._version === list._version) { + throw Error('Document has not been re-indexed in time'); + } + updatedOCCVersion = updatedList?._version; + */ + }; + + await waitUntilDocumentIndexed(checkIfListUpdated); + } else { + throw Error('No conversation has been updated'); + } + return getConversation(esClient, conversationIndex, existingConversation.id ?? ''); +}; + +export const transformToUpdateScheme = ( + updatedAt: string, + { + title, + apiConfig, + excludeFromLastConversationStorage, + messages, + replacements, + }: ConversationUpdateProps +) => { + return { + updated_at: updatedAt, + title, + api_config: { + connector_id: apiConfig?.connectorId, + connector_type_title: apiConfig?.connectorTypeTitle, + default_system_prompt_id: apiConfig?.defaultSystemPromptId, + model: apiConfig?.model, + provider: apiConfig?.provider, + }, + exclude_from_last_conversation_storage: excludeFromLastConversationStorage, + replacements, + messages: messages?.map((message) => ({ + '@timestamp': message.timestamp, + content: message.content, + is_error: message.isError, + presentation: message.presentation, + reader: message.reader, + replacements: message.replacements, + role: message.role, + trace_data: { + trace_id: message.traceData?.traceId, + transaction_id: message.traceData?.transactionId, + }, + })), + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts index 1b7a140ab2c14..8529a870a0243 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { PostActionsConnectorExecuteBodyInputs } from '../../schemas/post_actions_connector_execute'; +import { PostActionsConnectorExecuteBodyInputs } from '../../schemas/actions_connector/post_actions_connector_execute'; export type RequestBody = PostActionsConnectorExecuteBodyInputs; diff --git a/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts b/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts new file mode 100644 index 0000000000000..a00ce684b5cb7 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pRetry from 'p-retry'; + +// index.refresh_interval +// https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings +const DEFAULT_INDEX_REFRESH_TIME = 1000; + +/** + * retries until item has been re-indexed + * Data stream and using update_by_query, delete_by_query which do support only refresh=true/false, + * this utility needed response back when updates/delete applied + * @param fn execution function to retry + */ +export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise => { + await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); + await pRetry(fn, { + minTimeout: DEFAULT_INDEX_REFRESH_TIME, + retries: 5, + }); +}; diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index e2e79a2b4f1e8..3c212018ca642 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -39,6 +39,7 @@ import { } from './ai_assistant_service/lib/create_datastream'; import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; +import { registerConversationsRoutes } from './routes/register_routes'; export class ElasticAssistantPlugin implements @@ -111,6 +112,7 @@ export class ElasticAssistantPlugin // Evaluate postEvaluateRoute(router, getElserId); // Conversations + registerConversationsRoutes(router, this.logger); return { actions: plugins.actions, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts new file mode 100644 index 0000000000000..23723c4c7797c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + CONVERSATIONS_TABLE_MAX_PAGE_SIZE, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, +} from '../../../common/constants'; + +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { + BulkActionSkipResult, + BulkCrudActionResponse, + BulkCrudActionResults, + BulkCrudActionSummary, + PerformBulkActionRequestBody, + PerformBulkActionResponse, +} from '../../schemas/conversations/bulk_crud_conversations_route.gen'; +import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; + +export interface BulkOperationError { + message: string; + status?: number; + conversation: { + id: string; + name: string; + }; +} + +export type BulkActionError = BulkOperationError | unknown; + +const buildBulkResponse = ( + response: KibanaResponseFactory, + { + errors = [], + updated = [], + created = [], + deleted = [], + skipped = [], + }: { + errors?: BulkActionError[]; + updated?: ConversationResponse[]; + created?: ConversationResponse[]; + deleted?: string[]; + skipped?: BulkActionSkipResult[]; + } +): IKibanaResponse => { + const numSucceeded = updated.length + created.length + deleted.length; + const numSkipped = skipped.length; + const numFailed = errors.length; + + const summary: BulkCrudActionSummary = { + failed: numFailed, + succeeded: numSucceeded, + skipped: numSkipped, + total: numSucceeded + numFailed + numSkipped, + }; + + const results: BulkCrudActionResults = { + updated, + created, + deleted, + skipped, + }; + + if (numFailed > 0) { + return response.custom({ + headers: { 'content-type': 'application/json' }, + body: { + message: summary.succeeded > 0 ? 'Bulk edit partially failed' : 'Bulk edit failed', + status_code: 500, + attributes: { + errors: [], + results, + summary, + }, + }, + statusCode: 500, + }); + } + + const responseBody: BulkCrudActionResponse = { + success: true, + conversations_count: summary.total, + attributes: { results, summary }, + }; + + return response.ok({ body: responseBody }); +}; + +export const bulkActionConversationsRoute = ( + router: ElasticAssistantPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + options: { + tags: ['access:elasticAssistant'], + timeout: { + idleSocket: moment.duration(15, 'minutes').asMilliseconds(), + }, + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PerformBulkActionRequestBody), + }, + }, + }, + async (context, request, response): Promise> => { + const { body } = request; + const siemResponse = buildResponse(response); + + if (body?.update && body.update?.length > CONVERSATIONS_TABLE_MAX_PAGE_SIZE) { + return siemResponse.error({ + body: `More than ${CONVERSATIONS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, + statusCode: 400, + }); + } + + const abortController = new AbortController(); + + // subscribing to completed$, because it handles both cases when request was completed and aborted. + // when route is finished by timeout, aborted$ is not getting fired + request.events.completed$.subscribe(() => abortController.abort()); + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + const writer = await dataClient?.getWriter(); + + const { + errors, + docs_created: docsCreated, + docs_updated: docsUpdated, + docs_deleted: docsDeleted, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + } = await writer!.bulk({ + conversationsToCreate: body.create, + conversationsToDelete: body.delete?.ids, + conversationsToUpdate: body.update, + }); + + const created = await dataClient?.findConversations({ + page: 1, + perPage: 1000, + filter: docsCreated.map((updatedId) => `id:${updatedId}`).join(' OR '), + fields: ['id'], + }); + const updated = await dataClient?.findConversations({ + page: 1, + perPage: 1000, + filter: docsUpdated.map((updatedId) => `id:${updatedId}`).join(' OR '), + fields: ['id'], + }); + + return buildBulkResponse(response, { + updated: updated?.data, + created: created?.data, + deleted: docsDeleted ?? [], + errors, + }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts new file mode 100644 index 0000000000000..c9a640dde6d30 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, +} from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { + ConversationCreateProps, + ConversationResponse, +} from '../../schemas/conversations/common_attributes.gen'; +import { buildResponse } from '../utils'; +import { buildRouteValidationWithZod } from '../route_validation'; + +export const createConversationRoute = (router: ElasticAssistantPluginRouter): void => { + router.versioned + .post({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(ConversationCreateProps), + }, + }, + }, + async (context, request, response): Promise> => { + const siemResponse = buildResponse(response); + // const validationErrors = validateCreateRuleProps(request.body); + // if (validationErrors.length) { + // return siemResponse.error({ statusCode: 400, body: validationErrors }); + // } + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + if (request.body.id != null) { + const conversation = await dataClient?.getConversation(request.body.id); + if (conversation != null) { + return siemResponse.error({ + statusCode: 409, + body: `conversation with id: "${request.body.id}" already exists`, + }); + } + } + + const createdConversation = await dataClient?.createConversation(request.body); + return response.ok({ + body: ConversationResponse.parse(createdConversation), // transformValidate(createdConversation), + }); + } catch (err) { + const error = transformError(err as Error); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts new file mode 100644 index 0000000000000..8fe276f6da966 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, +} from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; +import { buildResponse } from '../utils'; + +export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .delete({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + params: schema.object({ + conversationId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise> => { + const siemResponse = buildResponse(response); + /* const validationErrors = validateQueryRuleByIds(request.query); + if (validationErrors.length) { + return siemResponse.error({ statusCode: 400, body: validationErrors }); + }*/ + + try { + const { conversationId } = request.params; + + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + const existingConversation = await dataClient?.getConversation(conversationId); + if (existingConversation == null) { + return siemResponse.error({ + body: `conversation id: "${conversationId}" not found`, + statusCode: 404, + }); + } + await dataClient?.deleteConversation(conversationId); + + return response.ok({ body: {} }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts new file mode 100644 index 0000000000000..8a013e9e68cf4 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, +} from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { + FindConversationsRequestQuery, + FindConversationsResponse, +} from '../../schemas/conversations/find_conversations_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; + +export const findConversationsRoute = (router: ElasticAssistantPluginRouter, logger: Logger) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + query: buildRouteValidationWithZod(FindConversationsRequestQuery), + }, + }, + }, + async (context, request, response): Promise> => { + const siemResponse = buildResponse(response); + + /* const validationErrors = validateFindConversationsRequestQuery(request.query); + if (validationErrors.length) { + return siemResponse.error({ statusCode: 400, body: validationErrors }); + }*/ + + try { + const { query } = request; + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + const result = await dataClient?.findConversations({ + perPage: query.per_page, + page: query.page, + sortField: query.sort_field, + sortOrder: query.sort_order, + filter: query.filter, + fields: query.fields, + }); + + return response.ok({ body: result }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts new file mode 100644 index 0000000000000..215c540d791d6 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, +} from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { + FindConversationsRequestQuery, + FindConversationsResponse, +} from '../../schemas/conversations/find_conversations_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; + +export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + query: buildRouteValidationWithZod(FindConversationsRequestQuery), + }, + }, + }, + async (context, request, response): Promise> => { + const siemResponse = buildResponse(response); + + /* const validationErrors = validateFindConversationsRequestQuery(request.query); + if (validationErrors.length) { + return siemResponse.error({ statusCode: 400, body: validationErrors }); + }*/ + + try { + const { query } = request; + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const currentUser = ctx.elasticAssistant.getCurrentUser(); + + const additionalFilter = query.filter ? `AND ${query.filter}` : ''; + const result = await dataClient?.findConversations({ + perPage: query.per_page, + page: query.page, + sortField: query.sort_field, + sortOrder: query.sort_order, + filter: `user.id:${currentUser?.profile_uid}${additionalFilter}`, + fields: query.fields, + }); + + return response.ok({ body: result }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts new file mode 100644 index 0000000000000..73782005ce7ff --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, +} from '../../../common/constants'; +import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; +import { buildResponse } from '../utils'; +import { ElasticAssistantPluginRouter } from '../../types'; + +export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + params: schema.object({ + conversationId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise> => { + const responseObj = buildResponse(response); + + const { conversationId } = request.params; + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const conversation = await dataClient?.getConversation(conversationId); + return response.ok({ body: conversation ?? {} }); + } catch (err) { + const error = transformError(err); + return responseObj.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts new file mode 100644 index 0000000000000..fdec45c464725 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, +} from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { + ConversationResponse, + ConversationUpdateProps, +} from '../../schemas/conversations/common_attributes.gen'; +import { buildResponse } from '../utils'; + +export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .put({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(ConversationUpdateProps), + params: schema.object({ + conversationId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise> => { + const siemResponse = buildResponse(response); + const { conversationId } = request.params; + /* const validationErrors = validateUpdateConversationProps(request.body); + if (validationErrors.length) { + return siemResponse.error({ statusCode: 400, body: validationErrors }); + }*/ + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + const existingConversation = await dataClient?.getConversation(conversationId); + if (existingConversation == null) { + return siemResponse.error({ + body: `conversation id: "${conversationId}" not found`, + statusCode: 404, + }); + } + const conversation = await dataClient?.updateConversation( + existingConversation, + request.body + ); + if (conversation == null) { + return siemResponse.error({ + body: `conversation id: "${conversationId}" was not updated`, + statusCode: 400, + }); + } + return response.ok({ + body: conversation, + }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/custom_http_request_error.ts b/x-pack/plugins/elastic_assistant/server/routes/custom_http_request_error.ts new file mode 100644 index 0000000000000..4b3ba1519e62c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/custom_http_request_error.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class CustomHttpRequestError extends Error { + constructor(message: string, public readonly statusCode: number = 500, meta?: unknown) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + this.message = message; + } +} diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index ed68f3526a112..d3b5e27105907 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -19,7 +19,7 @@ import { buildRouteValidation } from '../schemas/common'; import { PostActionsConnectorExecuteBody, PostActionsConnectorExecutePathParams, -} from '../schemas/post_actions_connector_execute'; +} from '../schemas/actions_connector/post_actions_connector_execute'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../types'; import { ESQL_RESOURCE } from './knowledge_base/constants'; import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts new file mode 100644 index 0000000000000..47ec048b35865 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +import { ElasticAssistantPluginRouter } from '../types'; +import { createConversationRoute } from './conversation/create_route'; +import { deleteConversationRoute } from './conversation/delete_route'; +import { findConversationsRoute } from './conversation/find_route'; +import { readConversationRoute } from './conversation/read_route'; +import { updateConversationRoute } from './conversation/update_route'; +import { findUserConversationsRoute } from './conversation/find_user_conversations_route'; +import { bulkActionConversationsRoute } from './conversation/bulk_actions_route'; + +export const registerConversationsRoutes = ( + router: ElasticAssistantPluginRouter, + logger: Logger +) => { + // Conversation CRUD + createConversationRoute(router); + readConversationRoute(router); + updateConversationRoute(router); + deleteConversationRoute(router); + + // Conversations bulk CRUD + bulkActionConversationsRoute(router, logger); + + // Conversations search + findConversationsRoute(router, logger); + findUserConversationsRoute(router); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 761552644feed..cd50f1f7f7c74 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -56,6 +56,8 @@ export class RequestContextFactory implements IRequestContextFactory { const getSpaceId = (): string => startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_NAMESPACE_STRING; + const getCurrentUser = () => startPlugins.security?.authc.getCurrentUser(request); + return { core: coreContext, @@ -67,6 +69,8 @@ export class RequestContextFactory implements IRequestContextFactory { getSpaceId, + getCurrentUser, + getAIAssistantSOClient: memoize(() => { const username = startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; @@ -77,12 +81,14 @@ export class RequestContextFactory implements IRequestContextFactory { }); }), - getAIAssistantDataClient: memoize(async () => - this.assistantService.createAIAssistantDatastreamClient({ + getAIAssistantDataClient: memoize(async () => { + const currentUser = getCurrentUser(); + return this.assistantService.createAIAssistantDatastreamClient({ namespace: getSpaceId(), logger: this.logger, - }) - ), + currentUser, + }); + }), }; } } diff --git a/x-pack/plugins/elastic_assistant/server/routes/route_validation.ts b/x-pack/plugins/elastic_assistant/server/routes/route_validation.ts new file mode 100644 index 0000000000000..58872422c5421 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/route_validation.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RouteValidationFunction, RouteValidationResultFactory } from '@kbn/core/server'; +import type { TypeOf, ZodType } from 'zod'; +import { stringifyZodError } from '@kbn/zod-helpers'; + +export const buildRouteValidationWithZod = + >(schema: T): RouteValidationFunction => + (inputValue: unknown, validationResult: RouteValidationResultFactory) => { + const decoded = schema.safeParse(inputValue); + if (decoded.success) { + return validationResult.ok(decoded.data); + } else { + return validationResult.badRequest(stringifyZodError(decoded.error)); + } + }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/utils.ts new file mode 100644 index 0000000000000..334e2f3712747 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/utils.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { snakeCase } from 'lodash/fp'; +import { BadRequestError } from '@kbn/securitysolution-es-utils'; + +import type { + RouteValidationFunction, + KibanaResponseFactory, + CustomHttpResponseOptions, +} from '@kbn/core/server'; +import { CustomHttpRequestError } from './custom_http_request_error'; + +export interface OutputError { + message: string; + statusCode: number; +} +export interface BulkError { + // Id can be single id or stringified ids. + id?: string; + error: { + status_code: number; + message: string; + }; +} + +export const createBulkErrorObject = ({ + id, + statusCode, + message, +}: { + id?: string; + statusCode: number; + message: string; +}): BulkError => { + if (id != null) { + return { + id, + error: { + status_code: statusCode, + message, + }, + }; + } else if (id != null) { + return { + id, + error: { + status_code: statusCode, + message, + }, + }; + } else { + return { + id: '(unknown id)', + error: { + status_code: statusCode, + message, + }, + }; + } +}; + +export const transformBulkError = ( + id: string | undefined, + err: Error & { statusCode?: number } +): BulkError => { + if (err instanceof CustomHttpRequestError) { + return createBulkErrorObject({ + id, + statusCode: err.statusCode ?? 400, + message: err.message, + }); + } else if (err instanceof BadRequestError) { + return createBulkErrorObject({ + id, + statusCode: 400, + message: err.message, + }); + } else { + return createBulkErrorObject({ + id, + statusCode: err.statusCode ?? 500, + message: err.message, + }); + } +}; + +interface Schema { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + validate: (input: any) => { value: any; error?: Error }; +} + +export const buildRouteValidation = + (schema: Schema): RouteValidationFunction => + (payload: T, { ok, badRequest }) => { + const { value, error } = schema.validate(payload); + if (error) { + return badRequest(error.message); + } + return ok(value); + }; + +const statusToErrorMessage = (statusCode: number) => { + switch (statusCode) { + case 400: + return 'Bad Request'; + case 401: + return 'Unauthorized'; + case 403: + return 'Forbidden'; + case 404: + return 'Not Found'; + case 409: + return 'Conflict'; + case 500: + return 'Internal Error'; + default: + return '(unknown error)'; + } +}; + +export class CustomResponseFactory { + constructor(private response: KibanaResponseFactory) {} + + error({ statusCode, body, headers }: CustomHttpResponseOptions) { + const contentType: CustomHttpResponseOptions['headers'] = { + 'content-type': 'application/json', + }; + const defaultedHeaders: CustomHttpResponseOptions['headers'] = { + ...contentType, + ...(headers ?? {}), + }; + + return this.response.custom({ + headers: defaultedHeaders, + statusCode, + body: Buffer.from( + JSON.stringify({ + message: body ?? statusToErrorMessage(statusCode), + status_code: statusCode, + }) + ), + }); + } +} + +export const buildResponse = (response: KibanaResponseFactory) => + new CustomResponseFactory(response); + +export const convertToSnakeCase = >( + obj: T +): Partial | null => { + if (!obj) { + return null; + } + return Object.keys(obj).reduce((acc, item) => { + const newKey = snakeCase(item); + return { ...acc, [newKey]: obj[item] }; + }, {}); +}; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/post_actions_connector_execute.ts rename to x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml b/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml deleted file mode 100644 index 20873b1a066da..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversation_apis.yml +++ /dev/null @@ -1,439 +0,0 @@ -openapi: 3.0.0 -info: - version: 1.0.0 - title: Elastic Assistant Conversation API - description: These APIs allow the consumer to manage conversations within Elastic Assistant. -paths: - /conversation: - post: - summary: Trigger calculation of Risk Scores - description: Calculates and persists a segment of Risk Scores, returning details about the calculation. - requestBody: - description: Details about the Risk Scores being calculated - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresCalculationRequest' - required: true - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresCalculationResponse' - '400': - description: Invalid request - /preview: - post: - summary: Preview the calculation of Risk Scores - description: Calculates and returns a list of Risk Scores, sorted by identifier_type and risk score. - requestBody: - description: Details about the Risk Scores being requested - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresPreviewRequest' - required: true - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresPreviewResponse' - '400': - description: Invalid request - /engine/status: - get: - summary: Get the status of the Risk Engine - description: Returns the status of both the legacy transform-based risk engine, as well as the new risk engine - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskEngineStatusResponse' - - /engine/init: - post: - summary: Initialize the Risk Engine - description: Initializes the Risk Engine by creating the necessary indices and mappings, removing old transforms, and starting the new risk engine - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskEngineInitResponse' - /engine/enable: - post: - summary: Enable the Risk Engine - requestBody: - content: - application/json: {} - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskEngineEnableResponse' - /engine/disable: - post: - summary: Disable the Risk Engine - requestBody: - content: - application/json: {} - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskEngineDisableResponse' - - -components: - schemas: - RiskScoresCalculationRequest: - type: object - required: - - data_view_id - - identifier_type - - range - properties: - after_keys: - description: Used to calculate a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. - allOf: - - $ref: '#/components/schemas/AfterKeys' - data_view_id: - $ref: '#/components/schemas/DataViewId' - description: The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. - debug: - description: If set to `true`, the internal ES requests/responses will be logged in Kibana. - type: boolean - filter: - $ref: '#/components/schemas/Filter' - description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores calculated. - page_size: - $ref: '#/components/schemas/PageSize' - identifier_type: - description: Used to restrict the type of risk scores calculated. - allOf: - - $ref: '#/components/schemas/IdentifierType' - range: - $ref: '#/components/schemas/DateRange' - description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. - weights: - $ref: '#/components/schemas/RiskScoreWeights' - - RiskScoresPreviewRequest: - type: object - required: - - data_view_id - properties: - after_keys: - description: Used to retrieve a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. - allOf: - - $ref: '#/components/schemas/AfterKeys' - data_view_id: - $ref: '#/components/schemas/DataViewId' - description: The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. - debug: - description: If set to `true`, a `debug` key is added to the response, containing both the internal request and response with elasticsearch. - type: boolean - filter: - $ref: '#/components/schemas/Filter' - description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores returned. - page_size: - $ref: '#/components/schemas/PageSize' - identifier_type: - description: Used to restrict the type of risk scores involved. If unspecified, both `host` and `user` scores will be returned. - allOf: - - $ref: '#/components/schemas/IdentifierType' - range: - $ref: '#/components/schemas/DateRange' - description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. - weights: - $ref: '#/components/schemas/RiskScoreWeights' - - RiskScoresCalculationResponse: - type: object - required: - - after_keys - - errors - - scores_written - properties: - after_keys: - description: Used to obtain the next "page" of risk scores. See also the `after_keys` key in a risk scores request. If this key is empty, the calculation is complete. - allOf: - - $ref: '#/components/schemas/AfterKeys' - errors: - type: array - description: A list of errors encountered during the calculation. - items: - type: string - scores_written: - type: number - format: integer - description: The number of risk scores persisted to elasticsearch. - - RiskScoresPreviewResponse: - type: object - required: - - after_keys - - scores - properties: - after_keys: - description: Used to obtain the next "page" of risk scores. See also the `after_keys` key in a risk scores request. If this key is empty, the calculation is complete. - allOf: - - $ref: '#/components/schemas/AfterKeys' - debug: - description: Object containing debug information, particularly the internal request and response from elasticsearch - type: object - properties: - request: - type: string - response: - type: string - scores: - type: array - description: A list of risk scores - items: - $ref: '#/components/schemas/RiskScore' - RiskEngineStatusResponse: - type: object - properties: - legacy_risk_engine_status: - $ref: '#/components/schemas/RiskEngineStatus' - risk_engine_status: - $ref: '#/components/schemas/RiskEngineStatus' - is_max_amount_of_risk_engines_reached: - description: Indicates whether the maximum amount of risk engines has been reached - type: boolean - RiskEngineInitResponse: - type: object - properties: - result: - type: object - properties: - risk_engine_enabled: - type: boolean - risk_engine_resources_installed: - type: boolean - risk_engine_configuration_created: - type: boolean - legacy_risk_engine_disabled: - type: boolean - errors: - type: array - items: - type: string - - - - RiskEngineEnableResponse: - type: object - properties: - success: - type: boolean - RiskEngineDisableResponse: - type: object - properties: - success: - type: boolean - - - AfterKeys: - type: object - properties: - host: - type: object - additionalProperties: - type: string - user: - type: object - additionalProperties: - type: string - example: - host: - 'host.name': 'example.host' - user: - 'user.name': 'example_user_name' - DataViewId: - description: The identifier of the Kibana data view to be used when generating risk scores. - example: security-solution-default - type: string - Filter: - description: An elasticsearch DSL filter object. Used to filter the risk inputs involved, which implicitly filters the risk scores themselves. - $ref: 'https://cloud.elastic.co/api/v1/api-docs/spec.json#/definitions/QueryContainer' - PageSize: - description: Specifies how many scores will be involved in a given calculation. Note that this value is per `identifier_type`, i.e. a value of 10 will calculate 10 host scores and 10 user scores, if available. To avoid missed data, keep this value consistent while paginating through scores. - default: 1000 - type: number - DateRange: - description: Defines the time period on which risk inputs will be filtered. - type: object - required: - - start - - end - properties: - start: - $ref: '#/components/schemas/KibanaDate' - end: - $ref: '#/components/schemas/KibanaDate' - KibanaDate: - type: string - oneOf: - - format: date - - format: date-time - - format: datemath - example: '2017-07-21T17:32:28Z' - IdentifierType: - type: string - enum: - - host - - user - RiskScore: - type: object - required: - - '@timestamp' - - id_field - - id_value - - calculated_level - - calculated_score - - calculated_score_norm - - category_1_score - - category_1_count - - inputs - properties: - '@timestamp': - type: string - format: 'date-time' - example: '2017-07-21T17:32:28Z' - description: The time at which the risk score was calculated. - id_field: - type: string - example: 'host.name' - description: The identifier field defining this risk score. Coupled with `id_value`, uniquely identifies the entity being scored. - id_value: - type: string - example: 'example.host' - description: The identifier value defining this risk score. Coupled with `id_field`, uniquely identifies the entity being scored. - calculated_level: - type: string - example: 'Critical' - description: Lexical description of the entity's risk. - calculated_score: - type: number - format: double - description: The raw numeric value of the given entity's risk score. - calculated_score_norm: - type: number - format: double - minimum: 0 - maximum: 100 - description: The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. - category_1_score: - type: number - format: double - description: The contribution of Category 1 to the overall risk score (`calculated_score`). Category 1 contains Detection Engine Alerts. - category_1_count: - type: number - format: integer - description: The number of risk input documents that contributed to the Category 1 score (`category_1_score`). - inputs: - type: array - description: A list of the highest-risk documents contributing to this risk score. Useful for investigative purposes. - items: - $ref: '#/components/schemas/RiskScoreInput' - - RiskScoreInput: - description: A generic representation of a document contributing to a Risk Score. - type: object - properties: - id: - type: string - example: 91a93376a507e86cfbf282166275b89f9dbdb1f0be6c8103c6ff2909ca8e1a1c - description: The unique identifier (`_id`) of the original source document - index: - type: string - example: .internal.alerts-security.alerts-default-000001 - description: The unique index (`_index`) of the original source document - category: - type: string - example: category_1 - description: The risk category of the risk input document. - description: - type: string - example: 'Generated from Detection Engine Rule: Malware Prevention Alert' - description: A human-readable description of the risk input document. - risk_score: - type: number - format: double - minimum: 0 - maximum: 100 - description: The weighted risk score of the risk input document. - timestamp: - type: string - example: '2017-07-21T17:32:28Z' - description: The @timestamp of the risk input document. - RiskScoreWeight: - description: "Configuration used to tune risk scoring. Weights can be used to change the score contribution of risk inputs for hosts and users at both a global level and also for Risk Input categories (e.g. 'category_1')." - type: object - required: - - type - properties: - type: - type: string - value: - type: string - host: - type: number - format: double - minimum: 0 - maximum: 1 - user: - type: number - format: double - minimum: 0 - maximum: 1 - example: - type: 'risk_category' - value: 'category_1' - host: 0.8 - user: 0.4 - RiskScoreWeights: - description: 'A list of weights to be applied to the scoring calculation.' - type: array - items: - $ref: '#/components/schemas/RiskScoreWeight' - example: - - type: 'risk_category' - value: 'category_1' - host: 0.8 - user: 0.4 - - type: 'global_identifier' - host: 0.5 - user: 0.1 - RiskEngineStatus: - type: string - enum: - - 'NOT_INSTALLED' - - 'DISABLED' - - 'ENABLED' - RiskEngineInitStep: - type: object - required: - - type - - success - properties: - type: - type: string - success: - type: boolean - error: - type: string - \ No newline at end of file diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts new file mode 100644 index 0000000000000..0c19e476f0206 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { BooleanFromString } from '@kbn/zod-helpers'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +import { + ConversationCreateProps, + ConversationUpdateProps, + ConversationResponse, +} from './common_attributes.gen'; + +export type BulkActionSkipReason = z.infer; +export const BulkActionSkipReason = z.literal('CONVERSATION_NOT_MODIFIED'); + +export type BulkActionSkipResult = z.infer; +export const BulkActionSkipResult = z.object({ + id: z.string(), + name: z.string().optional(), + skip_reason: BulkActionSkipReason, +}); + +export type ConversationDetailsInError = z.infer; +export const ConversationDetailsInError = z.object({ + id: z.string(), + name: z.string().optional(), +}); + +export type NormalizedConversationError = z.infer; +export const NormalizedConversationError = z.object({ + message: z.string(), + status_code: z.number().int(), + err_code: z.string().optional(), + conversations: z.array(ConversationDetailsInError), +}); + +export type BulkCrudActionResults = z.infer; +export const BulkCrudActionResults = z.object({ + updated: z.array(ConversationResponse), + created: z.array(ConversationResponse), + deleted: z.array(z.string()), + skipped: z.array(BulkActionSkipResult), +}); + +export type BulkCrudActionSummary = z.infer; +export const BulkCrudActionSummary = z.object({ + failed: z.number().int(), + skipped: z.number().int(), + succeeded: z.number().int(), + total: z.number().int(), +}); + +export type BulkCrudActionResponse = z.infer; +export const BulkCrudActionResponse = z.object({ + success: z.boolean().optional(), + status_code: z.number().int().optional(), + message: z.string().optional(), + conversations_count: z.number().int().optional(), + attributes: z.object({ + results: BulkCrudActionResults, + summary: BulkCrudActionSummary, + errors: z.array(NormalizedConversationError).optional(), + }), +}); + +export type BulkActionBase = z.infer; +export const BulkActionBase = z.object({ + /** + * Query to filter conversations + */ + query: z.string().optional(), + /** + * Array of conversation IDs + */ + ids: z.array(z.string()).min(1).optional(), +}); + +export type PerformBulkActionRequestQuery = z.infer; +export const PerformBulkActionRequestQuery = z.object({ + /** + * Enables dry run mode for the request call. + */ + dry_run: BooleanFromString.optional(), +}); +export type PerformBulkActionRequestQueryInput = z.input; + +export type PerformBulkActionRequestBody = z.infer; +export const PerformBulkActionRequestBody = z.object({ + delete: BulkActionBase.optional(), + create: z.array(ConversationCreateProps).optional(), + update: z.array(ConversationUpdateProps).optional(), +}); +export type PerformBulkActionRequestBodyInput = z.input; + +export type PerformBulkActionResponse = z.infer; +export const PerformBulkActionResponse = BulkCrudActionResponse; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml new file mode 100644 index 0000000000000..8385f04d0737f --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml @@ -0,0 +1,177 @@ +openapi: 3.0.0 +info: + title: Bulk Actions API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/conversations/_bulk_action: + post: + operationId: PerformBulkAction + x-codegen-enabled: true + summary: Applies a bulk action to multiple conversations + description: The bulk action is applied to all conversations that match the filter or to the list of conversations by their IDs. + tags: + - Bulk API + parameters: + - name: dry_run + in: query + description: Enables dry run mode for the request call. + required: false + schema: + type: boolean + requestBody: + content: + application/json: + schema: + type: object + properties: + delete: + $ref: '#/components/schemas/BulkActionBase' + create: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' + update: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/BulkCrudActionResponse' + +components: + schemas: + BulkActionSkipReason: + type: string + enum: + - CONVERSATION_NOT_MODIFIED + + BulkActionSkipResult: + type: object + properties: + id: + type: string + name: + type: string + skip_reason: + $ref: '#/components/schemas/BulkActionSkipReason' + required: + - id + - skip_reason + + ConversationDetailsInError: + type: object + properties: + id: + type: string + name: + type: string + required: + - id + + NormalizedConversationError: + type: object + properties: + message: + type: string + status_code: + type: integer + err_code: + type: string + conversations: + type: array + items: + $ref: '#/components/schemas/ConversationDetailsInError' + required: + - message + - status_code + - conversations + + BulkCrudActionResults: + type: object + properties: + updated: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + created: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + deleted: + type: array + items: + type: string + skipped: + type: array + items: + $ref: '#/components/schemas/BulkActionSkipResult' + required: + - updated + - created + - deleted + - skipped + + BulkCrudActionSummary: + type: object + properties: + failed: + type: integer + skipped: + type: integer + succeeded: + type: integer + total: + type: integer + required: + - failed + - skipped + - succeeded + - total + + BulkCrudActionResponse: + type: object + properties: + success: + type: boolean + status_code: + type: integer + message: + type: string + conversations_count: + type: integer + attributes: + type: object + properties: + results: + $ref: '#/components/schemas/BulkCrudActionResults' + summary: + $ref: '#/components/schemas/BulkCrudActionSummary' + errors: + type: array + items: + $ref: '#/components/schemas/NormalizedConversationError' + required: + - results + - summary + required: + - attributes + + + BulkActionBase: + x-inline: true + type: object + properties: + query: + type: string + description: Query to filter conversations + ids: + type: array + description: Array of conversation IDs + minItems: 1 + items: + type: string + \ No newline at end of file diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts new file mode 100644 index 0000000000000..558c3f50ae8c9 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +/** + * A string that is not empty and does not contain only whitespace + */ +export type NonEmptyString = z.infer; +export const NonEmptyString = z + .string() + .min(1) + .regex(/^(?! *$).+$/); + +/** + * A universally unique identifier + */ +export type UUID = z.infer; +export const UUID = z.string().uuid(); + +/** + * Could be any string, not necessarily a UUID + */ +export type User = z.infer; +export const User = z.object({ + /** + * User id. + */ + id: z.string().optional(), + /** + * User name. + */ + name: z.string().optional(), +}); + +/** + * Could be any string, not necessarily a UUID + */ +export type MessagePresentation = z.infer; +export const MessagePresentation = z.object({ + /** + * Could be any string, not necessarily a UUID + */ + delay: z.number().int().optional(), + /** + * Could be any string, not necessarily a UUID + */ + stream: z.boolean().optional(), +}); + +/** + * trace Data + */ +export type TraceData = z.infer; +export const TraceData = z.object({ + /** + * Could be any string, not necessarily a UUID + */ + transactionId: z.string().optional(), + /** + * Could be any string, not necessarily a UUID + */ + traceId: z.string().optional(), +}); + +export type Replacement = z.infer; +export const Replacement = z.object({}).catchall(z.unknown()); + +/** + * AI assistant sonversation message. + */ +export type Message = z.infer; +export const Message = z.object({ + /** + * Message content. + */ + content: z.string(), + /** + * Message content. + */ + reader: z.string().optional(), + replacements: z.array(Replacement).optional(), + /** + * Message role. + */ + role: z.enum(['system', 'user', 'assistant']), + /** + * The timestamp message was sent or received. + */ + timestamp: NonEmptyString, + /** + * Is error message. + */ + isError: z.boolean().optional(), + /** + * ID of the exception container + */ + presentation: MessagePresentation.optional(), + /** + * trace Data + */ + traceData: TraceData.optional(), +}); + +export type ApiConfig = z.infer; +export const ApiConfig = z.object({ + /** + * connector Id + */ + connectorId: z.string().optional(), + /** + * connector Type Title + */ + connectorTypeTitle: z.string().optional(), + /** + * defaultSystemPromptId + */ + defaultSystemPromptId: z.string().optional(), + /** + * Provider + */ + provider: z.enum(['OpenAI', 'Azure OpenAI']).optional(), + /** + * model + */ + model: z.string().optional(), +}); + +export type ErrorSchema = z.infer; +export const ErrorSchema = z + .object({ + id: UUID.optional(), + error: z.object({ + status_code: z.number().int().min(400), + message: z.string(), + }), + }) + .strict(); + +export type ConversationResponse = z.infer; +export const ConversationResponse = z.object({ + id: z.union([UUID, NonEmptyString]).optional(), + /** + * The conversation title. + */ + title: z.string().optional(), + timestamp: NonEmptyString.optional(), + /** + * The last time conversation was updated. + */ + updatedAt: z.string().optional(), + /** + * The last time conversation was updated. + */ + createdAt: z.string().optional(), + replacements: z.array(Replacement).optional(), + user: User.optional(), + /** + * The conversation messages. + */ + messages: z.array(Message).optional(), + /** + * LLM API configuration. + */ + apiConfig: ApiConfig.optional(), + /** + * Is default conversation. + */ + isDefault: z.boolean().optional(), + /** + * excludeFromLastConversationStorage. + */ + excludeFromLastConversationStorage: z.boolean().optional(), + /** + * Kibana space + */ + namespace: z.string().optional(), +}); + +export type ConversationUpdateProps = z.infer; +export const ConversationUpdateProps = z.object({ + id: z.union([UUID, NonEmptyString]), + /** + * The conversation title. + */ + title: z.string().optional(), + /** + * The conversation messages. + */ + messages: z.array(Message).optional(), + /** + * LLM API configuration. + */ + apiConfig: ApiConfig.optional(), + /** + * excludeFromLastConversationStorage. + */ + excludeFromLastConversationStorage: z.boolean().optional(), + replacements: z.array(Replacement).optional(), +}); + +export type ConversationCreateProps = z.infer; +export const ConversationCreateProps = z.object({ + id: z.union([UUID, NonEmptyString]).optional(), + /** + * The conversation title. + */ + title: z.string(), + /** + * The conversation messages. + */ + messages: z.array(Message).optional(), + /** + * LLM API configuration. + */ + apiConfig: ApiConfig.optional(), + /** + * Is default conversation. + */ + isDefault: z.boolean().optional(), + /** + * excludeFromLastConversationStorage. + */ + excludeFromLastConversationStorage: z.boolean().optional(), + replacements: z.array(Replacement).optional(), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml new file mode 100644 index 0000000000000..d3f5b887c3f4d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -0,0 +1,236 @@ +openapi: 3.0.0 +info: + title: Common Conversation Attributes + version: 'not applicable' +paths: {} +components: + x-codegen-enabled: true + schemas: + NonEmptyString: + type: string + pattern: ^(?! *$).+$ + minLength: 1 + description: A string that is not empty and does not contain only whitespace + + UUID: + type: string + format: uuid + description: A universally unique identifier + + User: + type: object + description: Could be any string, not necessarily a UUID + properties: + id: + type: string + description: User id. + name: + type: string + description: User name. + + MessagePresentation: + type: object + description: Could be any string, not necessarily a UUID + properties: + delay: + type: integer + description: Could be any string, not necessarily a UUID + stream: + type: boolean + description: Could be any string, not necessarily a UUID + + TraceData: + type: object + description: trace Data + properties: + transactionId: + type: string + description: Could be any string, not necessarily a UUID + traceId: + type: string + description: Could be any string, not necessarily a UUID + + Replacement: + type: object + additionalProperties: true + + Message: + type: object + description: AI assistant sonversation message. + required: + - 'timestamp' + - 'content' + - 'role' + properties: + content: + type: string + description: Message content. + reader: + type: string + description: Message content. + replacements: + type: array + items: + $ref: '#/components/schemas/Replacement' + role: + type: string + description: Message role. + enum: + - system + - user + - assistant + timestamp: + $ref: '#/components/schemas/NonEmptyString' + description: The timestamp message was sent or received. + isError: + type: boolean + description: Is error message. + presentation: + $ref: '#/components/schemas/MessagePresentation' + description: ID of the exception container + traceData: + $ref: '#/components/schemas/TraceData' + description: trace Data + + ApiConfig: + type: object + properties: + connectorId: + type: string + description: connector Id + connectorTypeTitle: + type: string + description: connector Type Title + defaultSystemPromptId: + type: string + description: defaultSystemPromptId + provider: + type: string + description: Provider + enum: + - OpenAI + - Azure OpenAI + model: + type: string + description: model + + ErrorSchema: + type: object + required: + - error + additionalProperties: false + properties: + id: + $ref: '#/components/schemas/UUID' + error: + type: object + required: + - status_code + - message + properties: + status_code: + type: integer + minimum: 400 + message: + type: string + + ConversationResponse: + type: object + properties: + id: + oneOf: + - $ref: '#/components/schemas/UUID' + - $ref: '#/components/schemas/NonEmptyString' + title: + type: string + description: The conversation title. + 'timestamp': + $ref: '#/components/schemas/NonEmptyString' + updatedAt: + description: The last time conversation was updated. + type: string + createdAt: + description: The last time conversation was updated. + type: string + replacements: + type: array + items: + $ref: '#/components/schemas/Replacement' + user: + $ref: '#/components/schemas/User' + messages: + type: array + items: + $ref: '#/components/schemas/Message' + description: The conversation messages. + apiConfig: + $ref: '#/components/schemas/ApiConfig' + description: LLM API configuration. + isDefault: + description: Is default conversation. + type: boolean + excludeFromLastConversationStorage: + description: excludeFromLastConversationStorage. + type: boolean + namespace: + type: string + description: Kibana space + + ConversationUpdateProps: + type: object + required: + - id + properties: + id: + oneOf: + - $ref: '#/components/schemas/UUID' + - $ref: '#/components/schemas/NonEmptyString' + title: + type: string + description: The conversation title. + messages: + type: array + items: + $ref: '#/components/schemas/Message' + description: The conversation messages. + apiConfig: + $ref: '#/components/schemas/ApiConfig' + description: LLM API configuration. + excludeFromLastConversationStorage: + description: excludeFromLastConversationStorage. + type: boolean + replacements: + type: array + items: + $ref: '#/components/schemas/Replacement' + + ConversationCreateProps: + type: object + required: + - title + properties: + id: + oneOf: + - $ref: '#/components/schemas/UUID' + - $ref: '#/components/schemas/NonEmptyString' + title: + type: string + description: The conversation title. + messages: + type: array + items: + $ref: '#/components/schemas/Message' + description: The conversation messages. + apiConfig: + $ref: '#/components/schemas/ApiConfig' + description: LLM API configuration. + isDefault: + description: Is default conversation. + type: boolean + excludeFromLastConversationStorage: + description: excludeFromLastConversationStorage. + type: boolean + replacements: + type: array + items: + $ref: '#/components/schemas/Replacement' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml new file mode 100644 index 0000000000000..8716d5a0d1e3b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml @@ -0,0 +1,97 @@ +openapi: 3.0.0 +info: + title: Create Conversation API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/conversations: + post: + operationId: CreateConversation + x-codegen-enabled: true + description: Create a conversation + tags: + - Conversation API + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + get: + operationId: ReadConversation + x-codegen-enabled: true + description: Read a single conversation + tags: + - Conversations API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + put: + operationId: UpdateConversation + x-codegen-enabled: true + description: Update a single conversation + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + delete: + operationId: DeleteConversation + x-codegen-enabled: true + description: Deletes a single conversation using the `id` field. + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts new file mode 100644 index 0000000000000..775fc93ae4059 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { ArrayFromString } from '@kbn/zod-helpers'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +import { ConversationResponse } from './common_attributes.gen'; + +export type FindConversationsSortField = z.infer; +export const FindConversationsSortField = z.enum([ + 'created_at', + 'is_default', + 'title', + 'updated_at', +]); +export type FindConversationsSortFieldEnum = typeof FindConversationsSortField.enum; +export const FindConversationsSortFieldEnum = FindConversationsSortField.enum; + +export type SortOrder = z.infer; +export const SortOrder = z.enum(['asc', 'desc']); +export type SortOrderEnum = typeof SortOrder.enum; +export const SortOrderEnum = SortOrder.enum; + +export type FindConversationsRequestQuery = z.infer; +export const FindConversationsRequestQuery = z.object({ + fields: ArrayFromString(z.string()).optional(), + /** + * Search query + */ + filter: z.string().optional(), + /** + * Field to sort by + */ + sort_field: FindConversationsSortField.optional(), + /** + * Sort order + */ + sort_order: SortOrder.optional(), + /** + * Page number + */ + page: z.coerce.number().int().min(1).optional().default(1), + /** + * Conversations per page + */ + per_page: z.coerce.number().int().min(0).optional().default(20), +}); +export type FindConversationsRequestQueryInput = z.input; + +export type FindConversationsResponse = z.infer; +export const FindConversationsResponse = z.object({ + page: z.number().int(), + perPage: z.number().int(), + total: z.number().int(), + data: z.array(ConversationResponse), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml new file mode 100644 index 0000000000000..33fccd5c7a671 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml @@ -0,0 +1,94 @@ +openapi: 3.0.0 +info: + title: Find Conversations API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/conversations/_find: + get: + operationId: FindConversations + x-codegen-enabled: true + description: Finds conversationss that match the given query. + tags: + - Conversations API + parameters: + - name: 'fields' + in: query + required: false + schema: + type: array + items: + type: string + - name: 'filter' + in: query + description: Search query + required: false + schema: + type: string + - name: 'sort_field' + in: query + description: Field to sort by + required: false + schema: + $ref: '#/components/schemas/FindConversationsSortField' + - name: 'sort_order' + in: query + description: Sort order + required: false + schema: + $ref: '#/components/schemas/SortOrder' + - name: 'page' + in: query + description: Page number + required: false + schema: + type: integer + minimum: 1 + default: 1 + - name: 'per_page' + in: query + description: Conversations per page + required: false + schema: + type: integer + minimum: 0 + default: 20 + + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + page: + type: integer + perPage: + type: integer + total: + type: integer + data: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + required: + - page + - perPage + - total + - data + +components: + schemas: + FindConversationsSortField: + type: string + enum: + - 'created_at' + - 'is_default' + - 'title' + - 'updated_at' + + SortOrder: + type: string + enum: + - 'asc' + - 'desc' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml new file mode 100644 index 0000000000000..8716d5a0d1e3b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml @@ -0,0 +1,97 @@ +openapi: 3.0.0 +info: + title: Create Conversation API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/conversations: + post: + operationId: CreateConversation + x-codegen-enabled: true + description: Create a conversation + tags: + - Conversation API + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + get: + operationId: ReadConversation + x-codegen-enabled: true + description: Read a single conversation + tags: + - Conversations API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + put: + operationId: UpdateConversation + x-codegen-enabled: true + description: Update a single conversation + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + delete: + operationId: DeleteConversation + x-codegen-enabled: true + description: Deletes a single conversation using the `id` field. + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml new file mode 100644 index 0000000000000..8716d5a0d1e3b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml @@ -0,0 +1,97 @@ +openapi: 3.0.0 +info: + title: Create Conversation API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/conversations: + post: + operationId: CreateConversation + x-codegen-enabled: true + description: Create a conversation + tags: + - Conversation API + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + get: + operationId: ReadConversation + x-codegen-enabled: true + description: Read a single conversation + tags: + - Conversations API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + put: + operationId: UpdateConversation + x-codegen-enabled: true + description: Update a single conversation + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + + /api/elastic_assistant/conversations/{id}: + delete: + operationId: DeleteConversation + x-codegen-enabled: true + description: Deletes a single conversation using the `id` field. + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index cf8c645aeebc2..9f863b8f4298f 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -13,6 +13,7 @@ import type { CoreRequestHandlerContext, CoreSetup, CustomRequestHandlerContext, + IRouter, KibanaRequest, Logger, SavedObjectsClientContract, @@ -20,9 +21,9 @@ import type { import { type MlPluginSetup } from '@kbn/ml-plugin/server'; import { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { AuthenticatedUser, SecurityPluginStart } from '@kbn/security-plugin/server'; import { AIAssistantSOClient } from './saved_object/ai_assistant_so_client'; -import { AIAssistantDataClient } from './ai_assistant_data_client'; +import { AIAssistantDataClient } from './conversations_data_client'; /** The plugin setup interface */ export interface ElasticAssistantPluginSetup { @@ -52,10 +53,10 @@ export interface ElasticAssistantApiRequestHandlerContext { logger: Logger; getServerBasePath: () => string; getSpaceId: () => string; + getCurrentUser: () => AuthenticatedUser | null; getAIAssistantDataClient: () => Promise; getAIAssistantSOClient: () => AIAssistantSOClient; } - /** * @internal */ @@ -63,6 +64,8 @@ export type ElasticAssistantRequestHandlerContext = CustomRequestHandlerContext< elasticAssistant: ElasticAssistantApiRequestHandlerContext; }>; +export type ElasticAssistantPluginRouter = IRouter; + export type ElasticAssistantPluginCoreSetupDependencies = CoreSetup< ElasticAssistantPluginStartDependencies, ElasticAssistantPluginStart @@ -113,7 +116,7 @@ export interface IIndexPatternString { } export interface PublicAIAssistantDataClient { - + getConversationsLimitValue: () => number; } export interface IAIAssistantDataClient { diff --git a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx index 61bd9fd3c937f..7d53b0c39d3e6 100644 --- a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx @@ -19,24 +19,28 @@ import { TIMELINE_CONVERSATION_TITLE } from './translations'; export const BASE_SECURITY_CONVERSATIONS: Record = { [ALERT_SUMMARY_CONVERSATION_ID]: { id: ALERT_SUMMARY_CONVERSATION_ID, + title: ALERT_SUMMARY_CONVERSATION_ID, isDefault: true, messages: [], apiConfig: {}, }, [DATA_QUALITY_DASHBOARD_CONVERSATION_ID]: { id: DATA_QUALITY_DASHBOARD_CONVERSATION_ID, + title: DATA_QUALITY_DASHBOARD_CONVERSATION_ID, isDefault: true, messages: [], apiConfig: {}, }, [DETECTION_RULES_CONVERSATION_ID]: { id: DETECTION_RULES_CONVERSATION_ID, + title: DETECTION_RULES_CONVERSATION_ID, isDefault: true, messages: [], apiConfig: {}, }, [EVENT_SUMMARY_CONVERSATION_ID]: { id: EVENT_SUMMARY_CONVERSATION_ID, + title: EVENT_SUMMARY_CONVERSATION_ID, isDefault: true, messages: [], apiConfig: {}, @@ -44,12 +48,14 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [TIMELINE_CONVERSATION_TITLE]: { excludeFromLastConversationStorage: true, id: TIMELINE_CONVERSATION_TITLE, + title: TIMELINE_CONVERSATION_TITLE, isDefault: true, messages: [], apiConfig: {}, }, [WELCOME_CONVERSATION_TITLE]: { id: WELCOME_CONVERSATION_TITLE, + title: WELCOME_CONVERSATION_TITLE, isDefault: true, theme: { title: ELASTIC_AI_ASSISTANT_TITLE, diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx index 8e6c9f20f7395..a942a1b8f0f4a 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx @@ -40,6 +40,7 @@ describe('getComments', () => { currentConversation: { apiConfig: {}, id: '1', + title: '1', messages: [ { role: user, diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 7a17a98bc0d6e..fd0dc532314a5 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useCallback } from 'react'; +import React from 'react'; import { i18n } from '@kbn/i18n'; import type { IToasts } from '@kbn/core-notifications-browser'; import { AssistantProvider as ElasticAssistantProvider } from '@kbn/elastic-assistant'; @@ -13,7 +13,7 @@ import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; import { augmentMessageCodeBlocks, LOCAL_STORAGE_KEY } from './helpers'; -import { useConversationStore } from './use_conversation_store'; +import { useBaseConversations } from './use_conversation_store'; import { DEFAULT_ALLOW, DEFAULT_ALLOW_REPLACEMENT } from './content/anonymization'; import { PROMPT_CONTEXTS } from './content/prompt_contexts'; import { BASE_SECURITY_QUICK_PROMPTS } from './content/quick_prompts'; @@ -42,11 +42,7 @@ export const AssistantProvider: React.FC = ({ children }) => { const isModelEvaluationEnabled = useIsExperimentalFeatureEnabled('assistantModelEvaluation'); const assistantStreamingEnabled = useIsExperimentalFeatureEnabled('assistantStreamingEnabled'); - const { conversations, setConversations } = useConversationStore(); - const getInitialConversation = useCallback(() => { - return conversations; - }, [conversations]); - + const baseConversations = useBaseConversations(); const assistantAvailability = useAssistantAvailability(); const assistantTelemetry = useAssistantTelemetry(); @@ -66,26 +62,25 @@ export const AssistantProvider: React.FC = ({ children }) => { alertsIndexPattern={alertsIndexPattern} augmentMessageCodeBlocks={augmentMessageCodeBlocks} assistantAvailability={assistantAvailability} - assistantTelemetry={assistantTelemetry} - defaultAllow={defaultAllow} - defaultAllowReplacement={defaultAllowReplacement} + assistantTelemetry={assistantTelemetry} // to server + defaultAllow={defaultAllow} // to server and plugin start + defaultAllowReplacement={defaultAllowReplacement} // to server and plugin start docLinks={{ ELASTIC_WEBSITE_URL, DOC_LINK_VERSION }} - baseAllow={DEFAULT_ALLOW} - baseAllowReplacement={DEFAULT_ALLOW_REPLACEMENT} + baseAllow={DEFAULT_ALLOW} // to server and plugin start + baseAllowReplacement={DEFAULT_ALLOW_REPLACEMENT} // to server and plugin start basePath={basePath} basePromptContexts={Object.values(PROMPT_CONTEXTS)} - baseQuickPrompts={BASE_SECURITY_QUICK_PROMPTS} - baseSystemPrompts={BASE_SECURITY_SYSTEM_PROMPTS} - getInitialConversations={getInitialConversation} + baseQuickPrompts={BASE_SECURITY_QUICK_PROMPTS} // to server and plugin start + baseSystemPrompts={BASE_SECURITY_SYSTEM_PROMPTS} // to server and plugin start + baseConversations={baseConversations} getComments={getComments} http={http} assistantStreamingEnabled={assistantStreamingEnabled} modelEvaluatorEnabled={isModelEvaluationEnabled} nameSpace={nameSpace} ragOnAlerts={ragOnAlerts} - setConversations={setConversations} - setDefaultAllow={setDefaultAllow} - setDefaultAllowReplacement={setDefaultAllowReplacement} + setDefaultAllow={setDefaultAllow} // remove + setDefaultAllowReplacement={setDefaultAllowReplacement} // remove title={ASSISTANT_TITLE} toasts={toasts} > diff --git a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx index 2a06b26a94420..2f621c13f7a68 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx @@ -11,7 +11,7 @@ import { useConversationStore } from '../use_conversation_store'; import { useKibana } from '../../common/lib/kibana'; export const useAssistantTelemetry = (): AssistantTelemetry => { - const { conversations } = useConversationStore(); + const conversations = useConversationStore(); const { services: { telemetry }, } = useKibana(); diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index e195f4542e29d..b942b20e1fef4 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -5,23 +5,16 @@ * 2.0. */ -import type { Conversation } from '@kbn/elastic-assistant'; +import { useFetchConversationsByUser, type Conversation } from '@kbn/elastic-assistant'; -import { unset } from 'lodash/fp'; +import { merge, unset } from 'lodash/fp'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; import { useMemo } from 'react'; -import { useLocalStorage } from '../../common/components/local_storage'; -import { LOCAL_STORAGE_KEY } from '../helpers'; import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { useLinkAuthorized } from '../../common/links'; import { SecurityPageName } from '../../../common'; -export interface UseConversationStore { - conversations: Record; - setConversations: React.Dispatch>>; -} - -export const useConversationStore = (): UseConversationStore => { +export const useConversationStore = (): Record => { const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); const baseConversations = useMemo( () => @@ -30,16 +23,27 @@ export const useConversationStore = (): UseConversationStore => { : unset(DATA_QUALITY_DASHBOARD_CONVERSATION_ID, BASE_SECURITY_CONVERSATIONS), [isDataQualityDashboardPageExists] ); - const [conversations, setConversations] = useLocalStorage>({ - defaultValue: baseConversations, - key: LOCAL_STORAGE_KEY, - isInvalidDefault: (valueFromStorage) => { - return !valueFromStorage; + + const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + const conversations = conversationsData?.data.reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; }, - }); + {} + ); - return { - conversations, - setConversations, - }; + return merge(baseConversations, conversations); +}; + +export const useBaseConversations = (): Record => { + const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); + const baseConversations = useMemo( + () => + isDataQualityDashboardPageExists + ? BASE_SECURITY_CONVERSATIONS + : unset(DATA_QUALITY_DASHBOARD_CONVERSATION_ID, BASE_SECURITY_CONVERSATIONS), + [isDataQualityDashboardPageExists] + ); + return baseConversations; }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index 3f65e542ed30c..09155cb5cf5af 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -158,7 +158,7 @@ const ActiveTimelineTab = memo( [activeTimelineTab] ); - const { conversations } = useConversationStore(); + const conversations = useConversationStore(); const hasTimelineConversationStarted = useMemo( () => conversations[TIMELINE_CONVERSATION_TITLE].messages.length > 0, From ed10f8b1c20cc6cdd37844eef65cd05914a95270 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 19 Dec 2023 10:09:30 -0800 Subject: [PATCH 003/141] fixed apis --- .../impl/assistant/api/conversations.ts | 69 ++++-------------- .../api/use_fetch_conversations_by_user.ts | 11 ++- .../assistant/chat_send/use_chat_send.tsx | 16 +++-- .../conversation_selector/index.tsx | 41 +++++++---- .../impl/assistant/index.test.tsx | 1 + .../impl/assistant/index.tsx | 70 ++++++++++++++++--- .../select_system_prompt/index.tsx | 1 + .../impl/assistant/use_conversation/index.tsx | 15 +++- .../connectorland/connector_setup/index.tsx | 38 ++++++---- .../server/__mocks__/request_context.ts | 2 +- .../create_conversation.ts | 4 +- .../find_conversations.ts | 3 +- .../get_conversation.ts | 3 +- .../conversations_data_client/transforms.ts | 52 +++++++------- .../update_conversation.ts | 23 +++--- .../elastic_assistant/server/plugin.ts | 9 +-- .../routes/conversation/create_route.ts | 13 +--- .../conversations/common_attributes.gen.ts | 1 - .../common_attributes.schema.yaml | 4 -- .../use_conversation_store/index.tsx | 37 +++++++--- 20 files changed, 242 insertions(+), 171 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts index c0eec6ca5599a..5a11435bfd652 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts @@ -10,12 +10,6 @@ import { HttpSetup } from '@kbn/core/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; import { Conversation, Message } from '../../assistant_context/types'; -export interface GetConversationsParams { - http: HttpSetup; - user?: string; - signal?: AbortSignal | undefined; -} - export interface GetConversationByIdParams { http: HttpSetup; id: string; @@ -23,44 +17,14 @@ export interface GetConversationByIdParams { } /** - * API call for getting current user conversations. + * API call for getting conversation by id. * * @param {Object} options - The options object. * @param {HttpSetup} options.http - HttpSetup + * @param {string} options.id - Conversation id. * @param {AbortSignal} [options.signal] - AbortSignal * - * @returns {Promise} - */ -export const getUserConversations = async ({ - http, - user, - signal, -}: GetConversationsParams): Promise => { - try { - const path = `/api/elastic_assistant/conversations`; - const response = await http.fetch(path, { - method: 'GET', - query: { - user, - }, - version: '2023-10-31', - signal, - }); - - return response as Conversation[]; - } catch (error) { - return error as IHttpFetchError; - } -}; - -/** - * API call for getting current user conversations. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} + * @returns {Promise} */ export const getConversationById = async ({ http, @@ -92,14 +56,14 @@ export interface PostConversationResponse { } /** - * API call for setting up the Knowledge Base. Provide a resource to set up a specific resource. + * API call for setting up the Conversation. * * @param {Object} options - The options object. * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.resource] - Resource to be added to the KB, otherwise sets up the base KB + * @param {Conversation} [options.conversation] - Conversation to be added * @param {AbortSignal} [options.signal] - AbortSignal * - * @returns {Promise} + * @returns {Promise} */ export const createConversationApi = async ({ http, @@ -131,14 +95,14 @@ export interface DeleteConversationResponse { } /** - * API call for deleting the Knowledge Base. Provide a resource to delete that specific resource. + * API call for deleting the Conversation. Provide a id to delete that specific resource. * * @param {Object} options - The options object. * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.id] - Resource to be deleted from the KB, otherwise delete the entire KB + * @param {string} [options.id] - Conversation id to be deleted * @param {AbortSignal} [options.signal] - AbortSignal * - * @returns {Promise} + * @returns {Promise} */ export const deleteConversationApi = async ({ http, @@ -162,6 +126,7 @@ export const deleteConversationApi = async ({ export interface PutConversationMessageParams { http: HttpSetup; conversationId: string; + title?: string; messages?: Message[]; apiConfig?: { connectorId?: string; @@ -174,23 +139,16 @@ export interface PutConversationMessageParams { signal?: AbortSignal | undefined; } -export interface PostEvaluationResponse { - evaluationId: string; - success: boolean; -} - /** * API call for evaluating models. * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.evalParams] - Params necessary for evaluation - * @param {AbortSignal} [options.signal] - AbortSignal + * @param {PutConversationMessageParams} options - The options object. * - * @returns {Promise} + * @returns {Promise} */ export const updateConversationApi = async ({ http, + title, conversationId, messages, apiConfig, @@ -203,6 +161,7 @@ export const updateConversationApi = async ({ method: 'PUT', body: JSON.stringify({ id: conversationId, + title, messages, replacements, apiConfig, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts index 714ad5869cb9b..adcd0064a505a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts @@ -34,11 +34,20 @@ export const useFetchConversationsByUser = () => { perPage: 100, }; - return useQuery([ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY, query], () => + const querySt = useQuery([ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY, query], () => http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { method: 'GET', version: AI_ASSISTANT_API_CURRENT_VERSION, query, }) ); + + const refresh = () => + http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { + method: 'GET', + version: AI_ASSISTANT_API_CURRENT_VERSION, + query, + }); + + return { ...querySt, refresh }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index eb284a5fdf35b..9c9639bfa3139 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -27,6 +27,7 @@ export interface UseChatSendProps { React.SetStateAction> >; setUserPrompt: React.Dispatch>; + refresh: () => Promise; } export interface UseChatSend { @@ -53,6 +54,7 @@ export const useChatSend = ({ setPromptTextPreview, setSelectedPromptContexts, setUserPrompt, + refresh, }: UseChatSendProps): UseChatSend => { const { isLoading, sendMessages } = useSendMessages(); const { appendMessage, appendReplacements, clearConversation, removeLastMessage } = @@ -101,7 +103,8 @@ export const useChatSend = ({ }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); - appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + await appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + await refresh(); }, [ allSystemPrompts, @@ -113,6 +116,7 @@ export const useChatSend = ({ currentConversation.replacements, editingSystemPromptId, http, + refresh, selectedPromptContexts, sendMessages, setPromptTextPreview, @@ -137,7 +141,8 @@ export const useChatSend = ({ replacements: currentConversation.replacements ?? {}, }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); - appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + await appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + await refresh(); }, [ appendMessage, appendReplacements, @@ -145,6 +150,7 @@ export const useChatSend = ({ currentConversation.id, currentConversation.replacements, http, + refresh, removeLastMessage, sendMessages, ]); @@ -157,7 +163,7 @@ export const useChatSend = ({ [handleSendMessage, setUserPrompt] ); - const handleOnChatCleared = useCallback(() => { + const handleOnChatCleared = useCallback(async () => { const defaultSystemPromptId = getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation, @@ -166,12 +172,14 @@ export const useChatSend = ({ setPromptTextPreview(''); setUserPrompt(''); setSelectedPromptContexts({}); - clearConversation(currentConversation.id); + await clearConversation(currentConversation.id); setEditingSystemPromptId(defaultSystemPromptId); + await refresh(); }, [ allSystemPrompts, clearConversation, currentConversation, + refresh, setEditingSystemPromptId, setPromptTextPreview, setSelectedPromptContexts, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 13252a8ea892e..104d849babfc9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -64,28 +64,43 @@ export const ConversationSelector: React.FC = React.memo( shouldDisableKeyboardShortcut = () => false, isDisabled = false, }) => { + const [conversations, setConversations] = useState>({}); const { allSystemPrompts, baseConversations } = useAssistantContext(); const { deleteConversation, createConversation } = useConversation(); - const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); - const conversations = merge( - baseConversations, - (conversationsData?.data ?? []).reduce>( - (transformed, conversation) => { + useEffect(() => { + if (!isLoading) { + const userConversations = (conversationsData?.data ?? []).reduce< + Record + >((transformed, conversation) => { transformed[conversation.id] = conversation; return transformed; - }, - {} - ) - ); + }, {}); + setConversations( + merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ) + ); + } + }, [baseConversations, conversationsData?.data, isLoading]); const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); const conversationOptions = useMemo(() => { return Object.values(conversations).map((conversation) => ({ value: { isDefault: conversation.isDefault ?? false }, - id: conversation.id ?? '', + id: conversation.id ?? conversation.title, label: conversation.title, })); }, [conversations]); @@ -150,10 +165,10 @@ export const ConversationSelector: React.FC = React.memo( const onChange = useCallback( (newOptions: ConversationSelectorOption[]) => { - if (newOptions.length === 0) { + if (newOptions.length === 0 || !newOptions?.[0].id) { setSelectedOptions([]); - } else if (conversationOptions.findIndex((o) => o.label === newOptions?.[0].label) !== -1) { - onConversationSelected(newOptions?.[0].label); + } else if (conversationOptions.findIndex((o) => o.id === newOptions?.[0].id) !== -1) { + onConversationSelected(newOptions?.[0].id); } }, [conversationOptions, onConversationSelected] diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index 3eb913b4d8e1c..09815198f592d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -36,6 +36,7 @@ const MOCK_CONVERSATION_TITLE = 'electric sheep'; const getInitialConversations = (): Record => ({ [WELCOME_CONVERSATION_TITLE]: { id: WELCOME_CONVERSATION_TITLE, + title: WELCOME_CONVERSATION_TITLE, messages: [], apiConfig: {}, }, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 14d0d3a1e28e9..abb56036b8598 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -94,25 +94,62 @@ const AssistantComponent: React.FC = ({ const [selectedPromptContexts, setSelectedPromptContexts] = useState< Record >({}); + const [conversations, setConversations] = useState>({}); const selectedPromptContextsCount = useMemo( () => Object.keys(selectedPromptContexts).length, [selectedPromptContexts] ); - const { amendMessage, getDefaultConversation } = useConversation(); + const { amendMessage, getDefaultConversation, getConversation } = useConversation(); + const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); - const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + useEffect(() => { + if (!isLoading) { + const userConversations = (conversationsData?.data ?? []).reduce< + Record + >((transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, {}); + setConversations( + merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ) + ); + } + }, [baseConversations, conversationsData?.data, isLoading]); - const conversations = merge( - baseConversations, - (conversationsData?.data ?? []).reduce>( + const refetchResults = useCallback(async () => { + const data = await refresh(); + const userConversations = (data.data ?? []).reduce>( (transformed, conversation) => { transformed[conversation.id] = conversation; return transformed; }, {} - ) - ); + ); + setConversations( + merge( + userConversations, + Object.keys(baseConversations) + .filter((baseId) => (data.data ?? []).find((c) => c.title === baseId) === undefined) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ) + ); + }, [baseConversations, refresh]); + // Connector details const { data: connectors, isSuccess: areConnectorsFetched } = useLoadConnectors({ http }); const defaultConnectorId = useMemo(() => getDefaultConnector(connectors)?.id, [connectors]); @@ -142,6 +179,21 @@ const AssistantComponent: React.FC = ({ } }, [selectedConversationId, setConversationId]); + /* const [currentConversation, setCurrentConversation] = useState( + conversations[selectedConversationId] ?? + getDefaultConversation({ conversationId: selectedConversationId }) + ); + + const refetchConversation = useCallback( + async (cId: string) => { + const updatedConversation = await getConversation(cId); + if (updatedConversation) { + setCurrentConversation(updatedConversation); + } + }, + [getConversation] + ); */ + const currentConversation = useMemo( () => conversations[selectedConversationId] ?? @@ -255,11 +307,12 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - (cId: string) => { + async (cId: string) => { setSelectedConversationId(cId); setEditingSystemPromptId( getDefaultSystemPrompt({ allSystemPrompts, conversation: conversations[cId] })?.id ); + // await refetchConversation(cId); }, [allSystemPrompts, conversations] ); @@ -376,6 +429,7 @@ const AssistantComponent: React.FC = ({ setEditingSystemPromptId, selectedPromptContexts, setSelectedPromptContexts, + refresh: refetchResults, }); const chatbotComments = useMemo( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx index 255cb59be0c20..fa01249a2b64d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx @@ -74,6 +74,7 @@ const SelectSystemPromptComponent: React.FC = ({ if (conversation) { setApiConfig({ conversationId: conversation.id, + title: conversation.title, apiConfig: { ...conversation.apiConfig, defaultSystemPromptId: prompt?.id, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index c12cb58fb22b6..c939db2435a6c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -82,11 +82,23 @@ interface UseConversation { removeLastMessage: (conversationId: string) => Promise; setApiConfig: ({ conversationId, apiConfig }: SetApiConfigProps) => void; createConversation: (conversation: Conversation) => Promise; + getConversation: (conversationId: string) => Promise; } export const useConversation = (): UseConversation => { const { allSystemPrompts, assistantTelemetry, http } = useAssistantContext(); + const getConversation = useCallback( + async (conversationId: string) => { + const currentConversation = await getConversationById({ http, id: conversationId }); + if (isHttpFetchError(currentConversation)) { + return; + } + return currentConversation; + }, + [http] + ); + /** * Removes the last message of conversation[] for a given conversationId */ @@ -146,9 +158,9 @@ export const useConversation = (): UseConversation => { if (isHttpFetchError(prevConversation)) { return; } + if (prevConversation != null) { messages = [...prevConversation.messages, message]; - await updateConversationApi({ http, conversationId, @@ -296,5 +308,6 @@ export const useConversation = (): UseConversation => { removeLastMessage, setApiConfig, createConversation, + getConversation, }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 7ce96eccbb415..0f6c202759ae4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useMemo, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { EuiCommentProps } from '@elastic/eui'; import { EuiAvatar, EuiBadge, EuiMarkdownFormat, EuiText, EuiTextAlign } from '@elastic/eui'; // eslint-disable-next-line @kbn/eslint/module_migration @@ -48,23 +48,38 @@ export const useConnectorSetup = ({ comments: EuiCommentProps[]; prompt: React.ReactElement; } => { + const [conversations, setConversations] = useState>({}); const { appendMessage, setApiConfig } = useConversation(); const bottomRef = useRef(null); // Access all conversations so we can add connector to all on initial setup const { actionTypeRegistry, http, baseConversations } = useAssistantContext(); - const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); - const conversations = merge( - baseConversations, - (conversationsData?.data ?? []).reduce>( - (transformed, conversationData) => { - transformed[conversation.id] = conversationData; + useEffect(() => { + if (!isLoading) { + const userConversations = (conversationsData?.data ?? []).reduce< + Record + >((transformed, conversationData) => { + transformed[conversationData.id] = conversationData; return transformed; - }, - {} - ) - ); + }, {}); + setConversations( + merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversationData) => { + transformed[conversationData] = baseConversations[conversationData]; + return transformed; + }, {}) + ) + ); + } + }, [baseConversations, conversationsData?.data, isLoading]); const { data: connectors, @@ -211,7 +226,6 @@ export const useConnectorSetup = ({ }, }); }); - console.log('setApiConfig ggg'); refetchConnectors?.(); setIsConnectorModalVisible(false); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index f6a1a71d674cd..e457923052e4e 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -80,7 +80,7 @@ const createElasticAssistantRequestContextMock = ( getCurrentUser: jest.fn(), getServerBasePath: jest.fn(), getSpaceId: jest.fn(), - core: jest.fn(), + core: clients.core, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index c5272a02edca0..6247a8a472943 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -71,7 +71,7 @@ export const createConversation = async ( const response = await esClient.create({ body, - id: conversation.isDefault && conversation.id ? conversation.id : uuidv4(), + id: uuidv4(), index: conversationIndex, refresh: 'wait_for', }); @@ -87,7 +87,6 @@ export const transformToCreateScheme = ( namespace: string, user: { id?: UUID; name?: string }, { - id, title, apiConfig, excludeFromLastConversationStorage, @@ -131,6 +130,7 @@ export const transformToCreateScheme = ( function transform(conversationSchema: CreateMessageSchema): ConversationResponse { const response: ConversationResponse = { + id: conversationSchema.id, timestamp: conversationSchema['@timestamp'], createdAt: conversationSchema.created_at, user: conversationSchema.user, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts index 5d242c09451a2..b3fefd1be1cf4 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts @@ -48,13 +48,12 @@ export const findConversations = async ({ track_total_hits: true, sort, }, - _source: false, + _source: true, from: (page - 1) * perPage, ignore_unavailable: true, index: conversationIndex, seq_no_primary_term: true, size: perPage, - fields: fields ?? ['*'], }); return { data: transformESToConversations(response), diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts index af995d5b41f62..7b1606d0d180f 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -23,8 +23,7 @@ export const getConversation = async ( }, }, }, - _source: false, - fields: ['*'], + _source: true, ignore_unavailable: true, index: conversationIndex, seq_no_primary_term: true, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index 1dfadfd81f425..d31fd3d72a22c 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -13,43 +13,47 @@ export const transformESToConversations = ( response: estypes.SearchResponse ): ConversationResponse[] => { return response.hits.hits.map((hit) => { - const conversationSchema = hit.fields; + const conversationSchema = hit._source; const conversation: ConversationResponse = { - timestamp: conversationSchema?.['@timestamp']?.[0], - createdAt: conversationSchema?.created_at?.[0], + timestamp: conversationSchema?.['@timestamp'], + createdAt: conversationSchema?.created_at, user: { - id: conversationSchema?.['user.id']?.[0], - name: conversationSchema?.['user.name']?.[0], + id: conversationSchema?.user?.id, + name: conversationSchema?.user?.name, }, - title: conversationSchema?.title?.[0], + title: conversationSchema?.title, apiConfig: { - connectorId: conversationSchema?.['api_config.connector_id']?.[0], - connectorTypeTitle: conversationSchema?.['api_config.connector_type_title']?.[0], - defaultSystemPromptId: conversationSchema?.['api_config.default_system_prompt_id']?.[0], - model: conversationSchema?.['api_config.model']?.[0], - provider: conversationSchema?.['api_config.provider']?.[0], + connectorId: conversationSchema?.api_config?.connector_id, + connectorTypeTitle: conversationSchema?.api_config?.connector_type_title, + defaultSystemPromptId: conversationSchema?.api_config?.default_system_prompt_id, + model: conversationSchema?.api_config?.model, + provider: conversationSchema?.api_config?.provider, }, excludeFromLastConversationStorage: - conversationSchema?.exclude_from_last_conversation_storage?.[0], - isDefault: conversationSchema?.is_default?.[0], + conversationSchema?.exclude_from_last_conversation_storage, + isDefault: conversationSchema?.is_default, messages: // eslint-disable-next-line @typescript-eslint/no-explicit-any conversationSchema?.messages?.map((message: Record) => ({ timestamp: message['@timestamp'], content: message.content, - isError: message.is_error, - presentation: message.presentation, - reader: message.reader, - replacements: message.replacements as Replacement[], + ...(message.is_error ? { isError: message.is_error } : {}), + ...(message.presentation ? { presentation: message.presentation } : {}), + ...(message.reader ? { reader: message.reader } : {}), + ...(message.replacements ? { replacements: message.replacements as Replacement[] } : {}), role: message.role, - traceData: { - traceId: message?.['trace_data.trace_id'], - transactionId: message?.['trace_data.transaction_id'], - }, + ...(message.trace_data + ? { + traceData: { + traceId: message.trace_data?.trace_id, + transactionId: message.trace_data?.transaction_id, + }, + } + : {}), })) ?? [], - updatedAt: conversationSchema?.updated_at?.[0], - replacements: conversationSchema?.replacements?.[0] as Replacement[], - namespace: conversationSchema?.namespace?.[0], + updatedAt: conversationSchema?.updated_at, + replacements: conversationSchema?.replacements as Replacement[], + namespace: conversationSchema?.namespace, id: hit._id, }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 6bfbd3b4cbe33..2dd0d81742025 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -73,23 +73,23 @@ export const updateConversation = async ( ...params, // when assigning undefined in painless, it will remove property and wil set it to null // for patch we don't want to remove unspecified value in payload - assignEmpty: !isPatch, + assignEmpty: !(isPatch ?? true), }, source: ` if (params.assignEmpty == true || params.containsKey('api_config')) { - if (params.assignEmpty == true || params.containsKey('api_config.connector_id')) { + if (params.assignEmpty == true || params.api_config.containsKey('connector_id')) { ctx._source.api_config.connector_id = params.api_config.connector_id; } - if (params.assignEmpty == true || params.containsKey('api_config.connector_type_title')) { + if (params.assignEmpty == true || params.api_config.containsKey('connector_type_title')) { ctx._source.api_config.connector_type_title = params.api_config.connector_type_title; } - if (params.assignEmpty == true || params.containsKey('api_config.default_system_prompt_id')) { + if (params.assignEmpty == true || params.api_config.containsKey('default_system_prompt_id')) { ctx._source.api_config.default_system_prompt_id = params.api_config.default_system_prompt_id; } - if (params.assignEmpty == true || params.containsKey('api_config.model')) { + if (params.assignEmpty == true || params.api_config.containsKey('model')) { ctx._source.api_config.model = params.api_config.model; } - if (params.assignEmpty == true || params.containsKey('api_config.provider')) { + if (params.assignEmpty == true || params.api_config.containsKey('provider')) { ctx._source.api_config.provider = params.api_config.provider; } } @@ -102,9 +102,8 @@ export const updateConversation = async ( if (params.assignEmpty == true || params.containsKey('title')) { ctx._source.title = params.title; } - ctx._source.updated_at = params.updated_at; if (params.assignEmpty == true || params.containsKey('messages')) { - ctx._source.messages = []; + def messages = []; for (message in params.messages) { def newMessage = [:]; newMessage['@timestamp'] = message['@timestamp']; @@ -113,12 +112,12 @@ export const updateConversation = async ( newMessage.presentation = message.presentation; newMessage.reader = message.reader; newMessage.replacements = message.replacements; - newMessage.role = message.role; - newMessage.trace_data.trace_id = message.trace_data.trace_id; - newMessage.trace_data.transaction_id = message.trace_data.transaction_id; - ctx._source.messages.add(enrichment); + newMessage.role = message.role; + messages.add(newMessage); } + ctx._source.messages = messages; } + ctx._source.updated_at = params.updated_at; `, }, }); diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index 8aeb87cc192e0..e6d6b3cd2b6a8 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -12,7 +12,6 @@ import { Logger, KibanaRequest, SavedObjectsClientContract, - CoreSetup, } from '@kbn/core/server'; import { once } from 'lodash'; @@ -42,13 +41,7 @@ import { import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; import { registerConversationsRoutes } from './routes/register_routes'; -import { appContextService, GetRegisteredTools } from './services/app_context'; - -interface CreateRouteHandlerContextParams { - core: CoreSetup; - logger: Logger; - getRegisteredTools: GetRegisteredTools; -} +import { appContextService } from './services/app_context'; export class ElasticAssistantPlugin implements diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts index c9a640dde6d30..c388b63a893b3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts @@ -49,20 +49,9 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); - - if (request.body.id != null) { - const conversation = await dataClient?.getConversation(request.body.id); - if (conversation != null) { - return siemResponse.error({ - statusCode: 409, - body: `conversation with id: "${request.body.id}" already exists`, - }); - } - } - const createdConversation = await dataClient?.createConversation(request.body); return response.ok({ - body: ConversationResponse.parse(createdConversation), // transformValidate(createdConversation), + body: ConversationResponse.parse(createdConversation), }); } catch (err) { const error = transformError(err as Error); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts index 558c3f50ae8c9..1b179c1abb072 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -210,7 +210,6 @@ export const ConversationUpdateProps = z.object({ export type ConversationCreateProps = z.infer; export const ConversationCreateProps = z.object({ - id: z.union([UUID, NonEmptyString]).optional(), /** * The conversation title. */ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml index d3f5b887c3f4d..120b135ece563 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -209,10 +209,6 @@ components: required: - title properties: - id: - oneOf: - - $ref: '#/components/schemas/UUID' - - $ref: '#/components/schemas/NonEmptyString' title: type: string description: The conversation title. diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index b942b20e1fef4..d47dbe2786a03 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -9,12 +9,13 @@ import { useFetchConversationsByUser, type Conversation } from '@kbn/elastic-ass import { merge, unset } from 'lodash/fp'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; -import { useMemo } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { useLinkAuthorized } from '../../common/links'; import { SecurityPageName } from '../../../common'; export const useConversationStore = (): Record => { + const [conversations, setConversations] = useState>({}); const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); const baseConversations = useMemo( () => @@ -24,14 +25,32 @@ export const useConversationStore = (): Record => { [isDataQualityDashboardPageExists] ); - const { data: conversationsData, isLoading } = useFetchConversationsByUser(); - const conversations = conversationsData?.data.reduce>( - (transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, - {} - ); + const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); + + useEffect(() => { + if (!isLoading) { + const userConversations = (conversationsData?.data ?? []).reduce< + Record + >((transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, {}); + setConversations( + merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ) + ); + } + }, [baseConversations, conversationsData?.data, isLoading]); return merge(baseConversations, conversations); }; From 04c76cc18e7acfd54e150bad36ddebfe78b29113 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 11 Jan 2024 19:17:11 -0800 Subject: [PATCH 004/141] changed client to use the new apis --- .../impl/assistant/api/conversations.ts | 8 +- ...> use_fetch_current_user_conversations.ts} | 30 +++-- .../impl/assistant/assistant_header/index.tsx | 5 +- .../assistant/assistant_overlay/index.tsx | 29 +++-- .../impl/assistant/assistant_title/index.tsx | 4 +- .../assistant/chat_send/use_chat_send.tsx | 2 +- .../conversation_selector/index.tsx | 28 +++-- .../conversation_selector_settings/index.tsx | 74 ++++++++--- .../translations.ts | 57 --------- .../impl/assistant/index.tsx | 115 +++++++++--------- .../system_prompt_settings.tsx | 2 +- .../use_settings_updater.tsx | 48 ++++++-- .../impl/assistant/use_conversation/index.tsx | 35 +++--- .../impl/assistant_context/index.tsx | 40 ------ .../impl/assistant_context/types.tsx | 1 + .../connector_selector_inline.tsx | 10 +- .../connectorland/connector_setup/index.tsx | 4 +- .../impl/connectorland/translations.ts | 2 +- .../impl/new_chat_by_id/index.tsx | 16 ++- .../packages/kbn-elastic-assistant/index.ts | 2 +- .../elastic_assistant/common/constants.ts | 2 + .../create_conversation.ts | 1 - .../find_conversations.ts | 6 + .../get_last_conversation.ts | 43 +++++++ .../server/conversations_data_client/index.ts | 22 ++++ .../elastic_assistant/server/plugin.ts | 38 ------ .../find_user_conversations_route.ts | 4 +- .../routes/conversation/read_last_route.ts | 50 ++++++++ .../server/routes/register_routes.ts | 2 + .../server/routes/request_context_factory.ts | 2 + .../public/assistant/provider.tsx | 12 +- .../use_conversation_store/index.tsx | 22 ++-- 32 files changed, 406 insertions(+), 310 deletions(-) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/api/{use_fetch_conversations_by_user.ts => use_fetch_current_user_conversations.ts} (69%) delete mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/translations.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts index 5a11435bfd652..7f2b374789592 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts @@ -51,10 +51,6 @@ export interface PostConversationParams { signal?: AbortSignal | undefined; } -export interface PostConversationResponse { - conversation: Conversation; -} - /** * API call for setting up the Conversation. * @@ -69,7 +65,7 @@ export const createConversationApi = async ({ http, conversation, signal, -}: PostConversationParams): Promise => { +}: PostConversationParams): Promise => { try { const path = `/api/elastic_assistant/conversations`; const response = await http.post(path, { @@ -78,7 +74,7 @@ export const createConversationApi = async ({ signal, }); - return response as PostConversationResponse; + return response as Conversation; } catch (error) { return error as IHttpFetchError; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts similarity index 69% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts index adcd0064a505a..1f343c514331f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_conversations_by_user.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts @@ -26,9 +26,8 @@ const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conv export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find_user` as const; -export const useFetchConversationsByUser = () => { +export const useFetchCurrentUserConversations = () => { const { http } = useKibana().services; - const query = { page: 1, perPage: 100, @@ -42,12 +41,29 @@ export const useFetchConversationsByUser = () => { }) ); - const refresh = () => - http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { + return { ...querySt }; +}; + +/** + * API call for getting conversation by id. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} options.id - Conversation id. + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const useLastConversation = () => { + const path = `/api/elastic_assistant/conversations/_last_user`; + const { http } = useKibana().services; + + const querySt = useQuery([path], () => + http.fetch(path, { method: 'GET', version: AI_ASSISTANT_API_CURRENT_VERSION, - query, - }); + }) + ); - return { ...querySt, refresh }; + return { ...querySt }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index e4c4454859d34..da454944692e7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -36,6 +36,7 @@ interface OwnProps { selectedConversationId: string; setIsSettingsModalVisible: React.Dispatch>; setSelectedConversationId: React.Dispatch>; + setCurrentConversation: React.Dispatch>; shouldDisableKeyboardShortcut?: () => boolean; showAnonymizedValues: boolean; title: string | JSX.Element; @@ -62,6 +63,7 @@ export const AssistantHeader: React.FC = ({ shouldDisableKeyboardShortcut, showAnonymizedValues, title, + setCurrentConversation, }) => { const showAnonymizedValuesChecked = useMemo( () => @@ -84,6 +86,7 @@ export const AssistantHeader: React.FC = ({ isDisabled={isDisabled} docLinks={docLinks} selectedConversation={currentConversation} + setCurrentConversation={setCurrentConversation} title={title} /> @@ -97,7 +100,7 @@ export const AssistantHeader: React.FC = ({ = 0; @@ -33,7 +34,21 @@ export const AssistantOverlay = React.memo(() => { WELCOME_CONVERSATION_TITLE ); const [promptContextId, setPromptContextId] = useState(); - const { assistantTelemetry, setShowAssistantOverlay, getConversationId } = useAssistantContext(); + const { assistantTelemetry, setShowAssistantOverlay } = useAssistantContext(); + + const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); + + const lastConversationId = useMemo(() => { + if (!isLoading) { + const sorted = conversationsData?.data.sort((convA, convB) => + convA.updatedAt && convB.updatedAt && convA.updatedAt > convB.updatedAt ? -1 : 1 + ); + if (sorted && sorted.length > 0) { + return sorted[0].id; + } + } + return WELCOME_CONVERSATION_TITLE; + }, [conversationsData?.data, isLoading]); // Bind `showAssistantOverlay` in SecurityAssistantContext to this modal instance const showOverlay = useCallback( @@ -43,7 +58,7 @@ export const AssistantOverlay = React.memo(() => { promptContextId: pid, conversationId: cid, }: ShowAssistantOverlayProps) => { - const newConversationId = getConversationId(cid); + const newConversationId = cid ?? lastConversationId; if (so) assistantTelemetry?.reportAssistantInvoked({ conversationId: newConversationId, @@ -54,7 +69,7 @@ export const AssistantOverlay = React.memo(() => { setPromptContextId(pid); setConversationId(newConversationId); }, - [assistantTelemetry, getConversationId] + [assistantTelemetry, lastConversationId] ); useEffect(() => { setShowAssistantOverlay(showOverlay); @@ -64,15 +79,15 @@ export const AssistantOverlay = React.memo(() => { const handleShortcutPress = useCallback(() => { // Try to restore the last conversation on shortcut pressed if (!isModalVisible) { - setConversationId(getConversationId()); + setConversationId(lastConversationId); assistantTelemetry?.reportAssistantInvoked({ invokedBy: 'shortcut', - conversationId: getConversationId(), + conversationId: lastConversationId, }); } setIsModalVisible(!isModalVisible); - }, [assistantTelemetry, isModalVisible, getConversationId]); + }, [isModalVisible, lastConversationId, assistantTelemetry]); // Register keyboard listener to show the modal when cmd + ; is pressed const onKeyDown = useCallback( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx index b3dcd0ae08429..45602d6b08180 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx @@ -32,7 +32,8 @@ export const AssistantTitle: React.FC<{ title: string | JSX.Element; docLinks: Omit; selectedConversation: Conversation | undefined; -}> = ({ isDisabled = false, title, docLinks, selectedConversation }) => { + setCurrentConversation: React.Dispatch>; +}> = ({ isDisabled = false, title, docLinks, selectedConversation, setCurrentConversation }) => { const selectedConnectorId = selectedConversation?.apiConfig?.connectorId; const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks; @@ -112,6 +113,7 @@ export const AssistantTitle: React.FC<{ isDisabled={isDisabled || selectedConversation === undefined} selectedConnectorId={selectedConnectorId} selectedConversation={selectedConversation} + setCurrentConversation={setCurrentConversation} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 9c9639bfa3139..16ddf0447aeb4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -27,7 +27,7 @@ export interface UseChatSendProps { React.SetStateAction> >; setUserPrompt: React.Dispatch>; - refresh: () => Promise; + refresh: () => Promise | undefined>; } export interface UseChatSend { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 104d849babfc9..2c5462d0e07ae 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -21,7 +21,7 @@ import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { merge } from 'lodash'; -import { Conversation, useFetchConversationsByUser } from '../../../..'; +import { Conversation, useFetchCurrentUserConversations } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import * as i18n from './translations'; import { DEFAULT_CONVERSATION_TITLE } from '../../use_conversation/translations'; @@ -34,7 +34,7 @@ interface Props { defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; selectedConversationId: string | undefined; - onConversationSelected: (conversationId: string) => void; + onConversationSelected: (conversationTitle: string, conversationId?: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; } @@ -69,7 +69,7 @@ export const ConversationSelector: React.FC = React.memo( const { deleteConversation, createConversation } = useConversation(); - const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); useEffect(() => { if (!isLoading) { @@ -111,7 +111,7 @@ export const ConversationSelector: React.FC = React.memo( // Callback for when user types to create a new system prompt const onCreateOption = useCallback( - (searchValue, flattenedOptions = []) => { + async (searchValue, flattenedOptions = []) => { if (!searchValue || !searchValue.trim().toLowerCase()) { return; } @@ -126,6 +126,7 @@ export const ConversationSelector: React.FC = React.memo( option.label.trim().toLowerCase() === normalizedSearchValue ) !== -1; + let cId; if (!optionExists) { const newConversation: Conversation = { id: searchValue, @@ -137,16 +138,16 @@ export const ConversationSelector: React.FC = React.memo( defaultSystemPromptId: defaultSystemPrompt?.id, }, }; - createConversation(newConversation); + cId = (await createConversation(newConversation))?.id; } - onConversationSelected(searchValue); + onConversationSelected(searchValue, cId); }, [ allSystemPrompts, + onConversationSelected, defaultConnectorId, defaultProvider, createConversation, - onConversationSelected, ] ); @@ -159,8 +160,17 @@ export const ConversationSelector: React.FC = React.memo( setTimeout(() => { deleteConversation(cId); }, 0); + const deletedConv = { ...conversations }; + delete deletedConv[cId]; + setConversations(deletedConv); }, - [conversationIds, deleteConversation, selectedConversationId, onConversationSelected] + [ + selectedConversationId, + conversations, + onConversationSelected, + conversationIds, + deleteConversation, + ] ); const onChange = useCallback( @@ -295,7 +305,7 @@ export const ConversationSelector: React.FC = React.memo( options={conversationOptions} selectedOptions={selectedOptions} onChange={onChange} - onCreateOption={onCreateOption} + onCreateOption={onCreateOption as unknown as () => void} renderOption={renderOption} compressed={true} isDisabled={isDisabled} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index e11b2cfede5f1..77692bc80f394 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -18,15 +18,19 @@ import { import React, { useCallback, useMemo, useState } from 'react'; import { css } from '@emotion/react'; +import useEvent from 'react-use/lib/useEvent'; import { Conversation } from '../../../..'; -import * as i18n from './translations'; +import * as i18n from '../conversation_selector/translations'; import { SystemPromptSelectorOption } from '../../prompt_editor/system_prompt/system_prompt_modal/system_prompt_selector/system_prompt_selector'; +const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; interface Props { conversations: Record; onConversationDeleted: (conversationId: string) => void; onConversationSelectionChange: (conversation?: Conversation | string) => void; selectedConversationId?: string; + shouldDisableKeyboardShortcut?: () => boolean; + isDisabled?: boolean; } const getPreviousConversationId = (conversationIds: string[], selectedConversationId = '') => { @@ -57,6 +61,8 @@ export const ConversationSelectorSettings: React.FC = React.memo( onConversationDeleted, onConversationSelectionChange, selectedConversationId, + isDisabled, + shouldDisableKeyboardShortcut = () => false, }) => { const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); @@ -153,6 +159,40 @@ export const ConversationSelectorSettings: React.FC = React.memo( handleSelectionChange(nextOption); }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); + // Register keyboard listener for quick conversation switching + const onKeyDown = useCallback( + (event: KeyboardEvent) => { + if (isDisabled || conversationIds.length <= 1) { + return; + } + + if ( + event.key === 'ArrowLeft' && + (isMac ? event.metaKey : event.ctrlKey) && + !shouldDisableKeyboardShortcut() + ) { + event.preventDefault(); + onLeftArrowClick(); + } + if ( + event.key === 'ArrowRight' && + (isMac ? event.metaKey : event.ctrlKey) && + !shouldDisableKeyboardShortcut() + ) { + event.preventDefault(); + onRightArrowClick(); + } + }, + [ + conversationIds.length, + isDisabled, + onLeftArrowClick, + onRightArrowClick, + shouldDisableKeyboardShortcut, + ] + ); + useEvent('keydown', onKeyDown); + const renderOption: ( option: ConversationSelectorSettingsOption, searchValue: string, @@ -218,6 +258,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( `} > = React.memo( onCreateOption={onCreateOption} renderOption={renderOption} compressed={true} + isDisabled={isDisabled} prepend={ - + + + } append={ - + + + } /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/translations.ts deleted file mode 100644 index 7ba634ded5f12..0000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/translations.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; - -export const SELECTED_CONVERSATION_LABEL = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.defaultConversationTitle', - { - defaultMessage: 'Conversations', - } -); - -export const CONVERSATION_SELECTOR_ARIA_LABEL = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.ariaLabel', - { - defaultMessage: 'Conversation selector', - } -); - -export const CONVERSATION_SELECTOR_PLACE_HOLDER = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.placeholderTitle', - { - defaultMessage: 'Select or type to create new...', - } -); - -export const CONVERSATION_SELECTOR_CUSTOM_OPTION_TEXT = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.CustomOptionTextTitle', - { - defaultMessage: 'Create new conversation:', - } -); - -export const PREVIOUS_CONVERSATION_TITLE = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.previousConversationTitle', - { - defaultMessage: 'Previous conversation', - } -); - -export const NEXT_CONVERSATION_TITLE = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.nextConversationTitle', - { - defaultMessage: 'Next conversation', - } -); - -export const DELETE_CONVERSATION = i18n.translate( - 'xpack.elasticAssistant.assistant.conversationSelectorSettings.deleteConversationTitle', - { - defaultMessage: 'Delete conversation', - } -); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 653203de8c33e..3e259993c73c1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -50,7 +50,10 @@ import { QuickPrompts } from './quick_prompts/quick_prompts'; import { useLoadConnectors } from '../connectorland/use_load_connectors'; import { useConnectorSetup } from '../connectorland/connector_setup'; import { ConnectorMissingCallout } from '../connectorland/connector_missing_callout'; -import { useFetchConversationsByUser } from './api/use_fetch_conversations_by_user'; +import { + useFetchCurrentUserConversations, + useLastConversation, +} from './api/use_fetch_current_user_conversations'; import { Conversation } from '../assistant_context/types'; export interface Props { @@ -84,8 +87,6 @@ const AssistantComponent: React.FC = ({ getComments, http, promptContexts, - setLastConversationId, - getConversationId, title, allSystemPrompts, baseConversations, @@ -100,8 +101,9 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); - const { amendMessage, getDefaultConversation, getConversation } = useConversation(); - const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); + const { amendMessage, getDefaultConversation } = useConversation(); + const { data: conversationsData, isLoading, refetch } = useFetchCurrentUserConversations(); + const { data: lastConversation, isLoading: isLoadingLast } = useLastConversation(); useEffect(() => { if (!isLoading) { @@ -129,26 +131,28 @@ const AssistantComponent: React.FC = ({ }, [baseConversations, conversationsData?.data, isLoading]); const refetchResults = useCallback(async () => { - const data = await refresh(); - const userConversations = (data.data ?? []).reduce>( - (transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, - {} - ); - setConversations( - merge( + const res = await refetch(); + if (!res.isLoading) { + const userConversations = (res?.data?.data ?? []).reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, + {} + ); + const updatedConv = merge( userConversations, Object.keys(baseConversations) - .filter((baseId) => (data.data ?? []).find((c) => c.title === baseId) === undefined) + .filter((baseId) => (res.data?.data ?? []).find((c) => c.title === baseId) === undefined) .reduce>((transformed, conversation) => { transformed[conversation] = baseConversations[conversation]; return transformed; }, {}) - ) - ); - }, [baseConversations, refresh]); + ); + setConversations(updatedConv); + return updatedConv; + } + }, [baseConversations, refetch]); // Connector details const { data: connectors, isSuccess: areConnectorsFetched } = useLoadConnectors({ http }); @@ -164,8 +168,15 @@ const AssistantComponent: React.FC = ({ [connectors] ); + const lastConversationId = useMemo(() => { + if (!isLoadingLast) { + return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; + } + return WELCOME_CONVERSATION_TITLE; + }, [isLoadingLast, lastConversation?.id]); + const [selectedConversationId, setSelectedConversationId] = useState( - isAssistantEnabled ? getConversationId(conversationId) : WELCOME_CONVERSATION_TITLE + isAssistantEnabled ? lastConversationId : WELCOME_CONVERSATION_TITLE ); useEffect(() => { @@ -174,27 +185,15 @@ const AssistantComponent: React.FC = ({ } }, [selectedConversationId, setConversationId]); - /* const [currentConversation, setCurrentConversation] = useState( - conversations[selectedConversationId] ?? - getDefaultConversation({ conversationId: selectedConversationId }) + const [currentConversation, setCurrentConversation] = useState( + getDefaultConversation({ conversationId: selectedConversationId }) ); - const refetchConversation = useCallback( - async (cId: string) => { - const updatedConversation = await getConversation(cId); - if (updatedConversation) { - setCurrentConversation(updatedConversation); - } - }, - [getConversation] - ); */ - - const currentConversation = useMemo( - () => - conversations[selectedConversationId] ?? - getDefaultConversation({ conversationId: selectedConversationId }), - [conversations, getDefaultConversation, selectedConversationId] - ); + useEffect(() => { + if (!isLoadingLast && lastConversation) { + setCurrentConversation(lastConversation); + } + }, [isLoadingLast, lastConversation]); // Welcome setup state const isWelcomeSetup = useMemo(() => { @@ -217,18 +216,6 @@ const AssistantComponent: React.FC = ({ // Settings modal state (so it isn't shared between assistant instances like Timeline) const [isSettingsModalVisible, setIsSettingsModalVisible] = useState(false); - // Remember last selection for reuse after keyboard shortcut is pressed. - // Clear it if there is no connectors - useEffect(() => { - if (areConnectorsFetched && !connectors?.length) { - return setLastConversationId(WELCOME_CONVERSATION_TITLE); - } - - if (!currentConversation.excludeFromLastConversationStorage) { - setLastConversationId(currentConversation.id); - } - }, [areConnectorsFetched, connectors?.length, currentConversation, setLastConversationId]); - const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ conversation: blockBotConversation, }); @@ -302,14 +289,25 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - async (cId: string) => { - setSelectedConversationId(cId); - setEditingSystemPromptId( - getDefaultSystemPrompt({ allSystemPrompts, conversation: conversations[cId] })?.id - ); - // await refetchConversation(cId); + async (cTitle: string, cId?: string) => { + if (conversations[cTitle] === undefined && cId) { + const updatedConv = await refetchResults(); + if (updatedConv) { + setCurrentConversation(updatedConv[cId]); + setSelectedConversationId(cId); + setEditingSystemPromptId( + getDefaultSystemPrompt({ allSystemPrompts, conversation: updatedConv[cId] })?.id + ); + } + } else { + setSelectedConversationId(cTitle); + setCurrentConversation(conversations[cTitle]); + setEditingSystemPromptId( + getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation })?.id + ); + } }, - [allSystemPrompts, conversations] + [allSystemPrompts, conversations, currentConversation, refetchResults] ); const handleOnSystemPromptSelectionChange = useCallback((systemPromptId?: string) => { @@ -516,6 +514,7 @@ const AssistantComponent: React.FC = ({ {showTitle && ( = React.memo( if (selectedSystemPrompt != null) { setUpdatedConversationSettings((prev) => keyBy( - 'id', + 'title', /* * updatedConversationWithPrompts calculates the present of prompt for * each conversation. Based on the values of selected conversation, it goes diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index e6305785cbae7..7b07895f2a3e9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -7,7 +7,7 @@ import React, { useCallback, useState } from 'react'; import { merge } from 'lodash'; -import { Conversation, Prompt, QuickPrompt, useFetchConversationsByUser } from '../../../..'; +import { Conversation, Prompt, QuickPrompt, useFetchCurrentUserConversations } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; import { bulkConversationsChange } from '../../api/use_bulk_actions_conversations'; @@ -38,7 +38,6 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { allQuickPrompts, allSystemPrompts, assistantTelemetry, - conversations, defaultAllow, defaultAllowReplacement, baseConversations, @@ -51,7 +50,7 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { http, } = useAssistantContext(); - const { data: conversationsData, isLoading } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); const conversations = merge( baseConversations, @@ -111,7 +110,28 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { const saveSettings = useCallback((): void => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - setConversations(updatedConversationSettings); + bulkConversationsChange(http, { + conversationsToUpdate: Object.keys(updatedConversationSettings).reduce( + (conversationsToUpdate: Conversation[], conversationId: string) => { + if (!updatedConversationSettings[conversationId].isDefault) { + conversationsToUpdate.push(updatedConversationSettings[conversationId]); + } + return conversationsToUpdate; + }, + [] + ), + conversationsToCreate: Object.keys(updatedConversationSettings).reduce( + (conversationsToCreate: Conversation[], conversationId: string) => { + if (updatedConversationSettings[conversationId].isDefault) { + conversationsToCreate.push(updatedConversationSettings[conversationId]); + } + return conversationsToCreate; + }, + [] + ), + conversationsToDelete: deletedConversationSettings, + }); + const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; const didUpdateRAGAlerts = @@ -130,20 +150,22 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { setDefaultAllow(updatedDefaultAllow); setDefaultAllowReplacement(updatedDefaultAllowReplacement); }, [ - assistantTelemetry, - knowledgeBase.isEnabledRAGAlerts, - knowledgeBase.isEnabledKnowledgeBase, setAllQuickPrompts, + updatedQuickPromptSettings, setAllSystemPrompts, - setDefaultAllow, - setDefaultAllowReplacement, - setKnowledgeBase, + updatedSystemPromptSettings, + http, updatedConversationSettings, + deletedConversationSettings, + knowledgeBase.isEnabledKnowledgeBase, + knowledgeBase.isEnabledRAGAlerts, + updatedKnowledgeBaseSettings, + setKnowledgeBase, + setDefaultAllow, updatedDefaultAllow, + setDefaultAllowReplacement, updatedDefaultAllowReplacement, - updatedKnowledgeBaseSettings, - updatedQuickPromptSettings, - updatedSystemPromptSettings, + assistantTelemetry, ]); return { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index b3204d307d5e1..ef7101f92fedd 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -7,7 +7,7 @@ import { useCallback } from 'react'; -import { isHttpFetchError } from '@kbn/core-http-browser'; +import { IHttpFetchError, isHttpFetchError } from '@kbn/core-http-browser'; import { useAssistantContext } from '../../assistant_context'; import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; @@ -80,13 +80,21 @@ interface UseConversation { getDefaultConversation: ({ conversationId, messages }: CreateConversationProps) => Conversation; deleteConversation: (conversationId: string) => void; removeLastMessage: (conversationId: string) => Promise; - setApiConfig: ({ conversationId, apiConfig }: SetApiConfigProps) => void; + setApiConfig: ({ + conversationId, + apiConfig, + }: SetApiConfigProps) => Promise>; createConversation: (conversation: Conversation) => Promise; getConversation: (conversationId: string) => Promise; } export const useConversation = (): UseConversation => { - const { allSystemPrompts, assistantTelemetry, http } = useAssistantContext(); + const { + allSystemPrompts, + assistantTelemetry, + knowledgeBase: { isEnabledKnowledgeBase, isEnabledRAGAlerts }, + http, + } = useAssistantContext(); const getConversation = useCallback( async (conversationId: string) => { @@ -98,12 +106,6 @@ export const useConversation = (): UseConversation => { }, [http] ); - const { - allSystemPrompts, - assistantTelemetry, - knowledgeBase: { isEnabledKnowledgeBase, isEnabledRAGAlerts }, - setConversations, - } = useAssistantContext(); /** * Removes the last message of conversation[] for a given conversationId @@ -157,7 +159,7 @@ export const useConversation = (): UseConversation => { * Append a message to the conversation[] for a given conversationId */ const appendMessage = useCallback( - ({ conversationId, message }: AppendMessageProps): Message[] => { + async ({ conversationId, message }: AppendMessageProps): Promise => { assistantTelemetry?.reportAssistantMessageSent({ conversationId, role: message.role, @@ -167,7 +169,7 @@ export const useConversation = (): UseConversation => { let messages: Message[] = []; const prevConversation = await getConversationById({ http, id: conversationId }); if (isHttpFetchError(prevConversation)) { - return; + return []; } if (prevConversation != null) { @@ -180,8 +182,7 @@ export const useConversation = (): UseConversation => { } return messages; }, - [assistantTelemetry, http] - [isEnabledKnowledgeBase, isEnabledRAGAlerts, assistantTelemetry, setConversations] + [assistantTelemetry, isEnabledKnowledgeBase, isEnabledRAGAlerts, http] ); const appendReplacements = useCallback( @@ -267,7 +268,7 @@ export const useConversation = (): UseConversation => { async (conversation: Conversation): Promise => { const response = await createConversationApi({ http, conversation }); if (!isHttpFetchError(response)) { - return response.conversation; + return response; } }, [http] @@ -287,9 +288,9 @@ export const useConversation = (): UseConversation => { * Update the apiConfig for a given conversationId */ const setApiConfig = useCallback( - async ({ conversationId, apiConfig, title, isDefault }: SetApiConfigProps): Promise => { + async ({ conversationId, apiConfig, title, isDefault }: SetApiConfigProps) => { if (isDefault && title === conversationId) { - await createConversationApi({ + return createConversationApi({ http, conversation: { apiConfig, @@ -300,7 +301,7 @@ export const useConversation = (): UseConversation => { }, }); } else { - await updateConversationApi({ + return updateConversationApi({ http, conversationId, apiConfig, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index b8d8f6a701cd1..58fbf8bec70fe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -89,9 +89,7 @@ export interface AssistantProviderProps { baseConversations: Record; modelEvaluatorEnabled?: boolean; nameSpace?: string; - ragOnAlerts?: boolean; // setConversations: React.Dispatch>>; - setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; title?: string; @@ -137,7 +135,6 @@ export interface UseAssistantContext { }) => EuiCommentProps[]; http: HttpSetup; knowledgeBase: KnowledgeBaseConfig; - getConversationId: (id?: string) => string; promptContexts: Record; modelEvaluatorEnabled: boolean; nameSpace: string; @@ -149,7 +146,6 @@ export interface UseAssistantContext { setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; setKnowledgeBase: React.Dispatch>; - setLastConversationId: React.Dispatch>; setSelectedSettingsTab: React.Dispatch>; setShowAssistantOverlay: (showAssistantOverlay: ShowAssistantOverlay) => void; showAssistantOverlay: ShowAssistantOverlay; @@ -182,8 +178,6 @@ export const AssistantProvider: React.FC = ({ baseConversations, modelEvaluatorEnabled = false, nameSpace = DEFAULT_ASSISTANT_NAMESPACE, - ragOnAlerts = false, - setConversations, setDefaultAllow, setDefaultAllowReplacement, title = DEFAULT_ASSISTANT_TITLE, @@ -261,37 +255,6 @@ export const AssistantProvider: React.FC = ({ */ const [selectedSettingsTab, setSelectedSettingsTab] = useState(CONVERSATIONS_TAB); - const [conversations, setConversationsInternal] = useState(getInitialConversations()); - const conversationIds = useMemo(() => Object.keys(conversations).sort(), [conversations]); - - // TODO: This is a fix for conversations not loading out of localstorage. Also re-introduces our cascading render issue (as it loops back in localstorage) - useEffect(() => { - setConversationsInternal(getInitialConversations()); - }, [getInitialConversations]); - - const onConversationsUpdated = useCallback< - React.Dispatch>> - >( - ( - newConversations: - | Record - | ((prev: Record) => Record) - ) => { - if (typeof newConversations === 'function') { - const updater = newConversations; - setConversationsInternal((prevValue) => { - const newValue = updater(prevValue); - setConversations(newValue); - return newValue; - }); - } else { - setConversations(newConversations); - setConversationsInternal(newConversations); - } - }, - [setConversations] - ); - const getConversationId = useCallback( // if a conversationId has been provided, use that // if not, check local storage @@ -339,7 +302,6 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, - getConversationId, setLastConversationId: setLocalStorageLastConversationId, baseConversations, }), @@ -362,12 +324,10 @@ export const AssistantProvider: React.FC = ({ getComments, http, localStorageKnowledgeBase, - getConversationId, localStorageQuickPrompts, localStorageSystemPrompts, modelEvaluatorEnabled, nameSpace, - // onConversationsUpdated, promptContexts, registerPromptContext, selectedSettingsTab, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index f99e7c2be5447..09f99d55440ae 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -65,6 +65,7 @@ export interface Conversation { id: string; title: string; messages: Message[]; + updatedAt?: string; replacements?: Record; theme?: ConversationTheme; isDefault?: boolean; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx index c7ca37b491738..4f9069b5d7840 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx @@ -23,6 +23,7 @@ interface Props { isDisabled?: boolean; selectedConnectorId?: string; selectedConversation?: Conversation; + setCurrentConversation: React.Dispatch>; } const inputContainerClassName = css` @@ -65,7 +66,7 @@ const placeholderButtonClassName = css` * A compact wrapper of the ConnectorSelector component used in the Settings modal. */ export const ConnectorSelectorInline: React.FC = React.memo( - ({ isDisabled = false, selectedConnectorId, selectedConversation }) => { + ({ isDisabled = false, selectedConnectorId, selectedConversation, setCurrentConversation }) => { const [isOpen, setIsOpen] = useState(false); const { actionTypeRegistry, assistantAvailability, http } = useAssistantContext(); const { setApiConfig } = useConversation(); @@ -93,7 +94,7 @@ export const ConnectorSelectorInline: React.FC = React.memo( }, [isOpen]); const onChange = useCallback( - (connector: AIConnector) => { + async (connector: AIConnector) => { const connectorId = connector.id; if (connectorId === ADD_NEW_CONNECTOR) { return; @@ -105,7 +106,7 @@ export const ConnectorSelectorInline: React.FC = React.memo( setIsOpen(false); if (selectedConversation != null) { - setApiConfig({ + const res = await setApiConfig({ conversationId: selectedConversation.id, title: selectedConversation.title, isDefault: selectedConversation.isDefault, @@ -118,9 +119,10 @@ export const ConnectorSelectorInline: React.FC = React.memo( model: model ?? config?.defaultModel, }, }); + setCurrentConversation(res as Conversation); } }, - [selectedConversation, setApiConfig] + [selectedConversation, setApiConfig, setCurrentConversation] ); return ( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 0f6c202759ae4..4026cab6923ae 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -16,7 +16,7 @@ import { ActionType } from '@kbn/triggers-actions-ui-plugin/public'; import { merge } from 'lodash/fp'; import { AddConnectorModal } from '../add_connector_modal'; import { WELCOME_CONVERSATION } from '../../assistant/use_conversation/sample_conversations'; -import { Conversation, Message, useFetchConversationsByUser } from '../../..'; +import { Conversation, Message, useFetchCurrentUserConversations } from '../../..'; import { useLoadActionTypes } from '../use_load_action_types'; import { StreamingText } from '../../assistant/streaming_text'; import { ConnectorButton } from '../connector_button'; @@ -54,7 +54,7 @@ export const useConnectorSetup = ({ // Access all conversations so we can add connector to all on initial setup const { actionTypeRegistry, http, baseConversations } = useAssistantContext(); - const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); useEffect(() => { if (!isLoading) { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/translations.ts index 140f5faa2b656..9d1a5d0c58a80 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/translations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/translations.ts @@ -41,7 +41,7 @@ export const PRECONFIGURED_CONNECTOR = i18n.translate( export const CONNECTOR_SELECTOR_TITLE = i18n.translate( 'xpack.elasticAssistant.assistant.connectors.connectorSelector.ariaLabel', { - defaultMessage: 'Conversation Selector', + defaultMessage: 'Connector Selector', } ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx index 092e48d6f0bfd..4e52018fa9c63 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx @@ -34,15 +34,13 @@ const NewChatByIdComponent: React.FC = ({ const { showAssistantOverlay } = useAssistantContext(); // proxy show / hide calls to assistant context, using our internal prompt context id: - const showOverlay = useCallback( - () => - showAssistantOverlay({ - conversationId, - promptContextId, - showOverlay: true, - }), - [conversationId, promptContextId, showAssistantOverlay] - ); + const showOverlay = useCallback(() => { + showAssistantOverlay({ + conversationId, + promptContextId, + showOverlay: true, + }); + }, [conversationId, promptContextId, showAssistantOverlay]); const icon = useMemo(() => { if (iconType === null) { diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index a671d1189754b..5e69f84b65698 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -141,4 +141,4 @@ export type { DeleteKnowledgeBaseResponse } from './impl/assistant/api'; export type { GetKnowledgeBaseStatusResponse } from './impl/assistant/api'; export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; -export { useFetchConversationsByUser } from './impl/assistant/api/use_fetch_conversations_by_user'; +export { useFetchCurrentUserConversations } from './impl/assistant/api/use_fetch_current_user_conversations'; diff --git a/x-pack/plugins/elastic_assistant/common/constants.ts b/x-pack/plugins/elastic_assistant/common/constants.ts index 13712b169b840..920e7c4d170e8 100755 --- a/x-pack/plugins/elastic_assistant/common/constants.ts +++ b/x-pack/plugins/elastic_assistant/common/constants.ts @@ -25,6 +25,8 @@ export const ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_last_user`; + export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_CREATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_create`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_DELETE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_delete`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_UPDATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_update`; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 6247a8a472943..afdc827c1b9b6 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -130,7 +130,6 @@ export const transformToCreateScheme = ( function transform(conversationSchema: CreateMessageSchema): ConversationResponse { const response: ConversationResponse = { - id: conversationSchema.id, timestamp: conversationSchema['@timestamp'], createdAt: conversationSchema.created_at, user: conversationSchema.user, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts index b3fefd1be1cf4..6f62082643d80 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts @@ -41,6 +41,12 @@ export const findConversations = async ({ const ascOrDesc = sortOrder ?? ('asc' as const); if (sortField != null) { sort = [{ [sortField]: ascOrDesc }]; + } else { + sort = { + updated_at: { + order: 'desc', + }, + }; } const response = await esClient.search({ body: { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts new file mode 100644 index 0000000000000..c6d709eb2214e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; + +import { ConversationResponse, UUID } from '../schemas/conversations/common_attributes.gen'; +import { SearchEsConversationSchema } from './types'; +import { transformESToConversations } from './transforms'; + +export const getLastConversation = async ( + esClient: ElasticsearchClient, + conversationIndex: string, + userId: UUID +): Promise => { + const response = await esClient.search({ + body: { + sort: { + updated_at: { + order: 'desc', + }, + }, + query: { + bool: { + filter: [ + { term: { 'user.id': userId } }, + { term: { excludeFromLastConversationStorage: false } }, + ], + }, + }, + size: 1, + }, + _source: true, + ignore_unavailable: true, + index: conversationIndex, + seq_no_primary_term: true, + }); + const conversation = transformESToConversations(response); + return conversation[0] ?? null; +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 4e9abe8252b64..7c68bf057f3f4 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -25,6 +25,7 @@ import { findConversations } from './find_conversations'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; import { deleteConversation } from './delete_conversation'; +import { getLastConversation } from './get_last_conversation'; export enum OpenAiProviderType { OpenAi = 'OpenAI', @@ -120,6 +121,27 @@ export class AIAssistantDataClient { return getConversation(esClient, this.indexTemplateAndPattern.alias, id); }; + /** + * Creates a conversation, if given at least the "title" and "apiConfig" + * See {@link https://www.elastic.co/guide/en/security/current/} + * for more information around formats of the deserializer and serializer + * @param options + * @param options.id The id of the conversat to create or "undefined" if you want an "id" to be auto-created for you + * @param options.title A custom deserializer for the conversation. Optionally, you an define this as handle bars. See online docs for more information. + * @param options.messages Set this to true if this is a conversation that is "immutable"/"pre-packaged". + * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. + * @returns The conversation created + */ + public getLastConversation = async (): Promise => { + const { currentUser } = this; + const esClient = await this.options.elasticsearchClientPromise; + return getLastConversation( + esClient, + this.indexTemplateAndPattern.alias, + currentUser?.profile_uid ?? '' + ); + }; + public findConversations = async ({ perPage, page, diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index c2947d7defde8..3d26520968a95 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -12,7 +12,6 @@ import { Logger, KibanaRequest, SavedObjectsClientContract, - type AnalyticsServiceSetup, } from '@kbn/core/server'; import { once } from 'lodash'; @@ -44,14 +43,6 @@ import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; import { registerConversationsRoutes } from './routes/register_routes'; import { appContextService } from './services/app_context'; -import { appContextService, GetRegisteredTools } from './services/app_context'; - -interface CreateRouteHandlerContextParams { - core: CoreSetup; - logger: Logger; - getRegisteredTools: GetRegisteredTools; - telemetry: AnalyticsServiceSetup; -} export class ElasticAssistantPlugin implements @@ -73,27 +64,6 @@ export class ElasticAssistantPlugin this.kibanaVersion = initializerContext.env.packageInfo.version; } - private createRouteHandlerContext = ({ - core, - logger, - getRegisteredTools, - telemetry, - }: CreateRouteHandlerContextParams): IContextProvider< - ElasticAssistantRequestHandlerContext, - typeof PLUGIN_ID - > => { - return async function elasticAssistantRouteHandlerContext(context, request) { - const [_, pluginsStart] = await core.getStartServices(); - - return { - actions: pluginsStart.actions, - getRegisteredTools, - logger, - telemetry, - }; - }; - }; - public setup( core: ElasticAssistantPluginCoreSetupDependencies, plugins: ElasticAssistantPluginSetupDependencies @@ -122,14 +92,6 @@ export class ElasticAssistantPlugin core.http.registerRouteHandlerContext( PLUGIN_ID, (context, request) => requestContextFactory.create(context, request) - this.createRouteHandlerContext({ - core: core as CoreSetup, - logger: this.logger, - getRegisteredTools: (pluginName: string) => { - return appContextService.getRegisteredTools(pluginName); - }, - telemetry: core.analytics, - }) ); events.forEach((eventConfig) => core.analytics.registerEventType(eventConfig)); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts index 215c540d791d6..bab6e3f9b68e3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts @@ -39,7 +39,7 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) }, }, async (context, request, response): Promise> => { - const siemResponse = buildResponse(response); + const assistantResponse = buildResponse(response); /* const validationErrors = validateFindConversationsRequestQuery(request.query); if (validationErrors.length) { @@ -65,7 +65,7 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) return response.ok({ body: result }); } catch (err) { const error = transformError(err); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts new file mode 100644 index 0000000000000..0a5c06e31702e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST, +} from '../../../common/constants'; +import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; +import { buildResponse } from '../utils'; +import { ElasticAssistantPluginRouter } from '../../types'; + +export const readLastConversationRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: false, + }, + async (context, request, response): Promise> => { + const responseObj = buildResponse(response); + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + + const conversation = await dataClient?.getLastConversation(); + return response.ok({ body: conversation ?? {} }); + } catch (err) { + const error = transformError(err); + return responseObj.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index 47ec048b35865..e9a178baf6a35 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -15,6 +15,7 @@ import { readConversationRoute } from './conversation/read_route'; import { updateConversationRoute } from './conversation/update_route'; import { findUserConversationsRoute } from './conversation/find_user_conversations_route'; import { bulkActionConversationsRoute } from './conversation/bulk_actions_route'; +import { readLastConversationRoute } from './conversation/read_last_route'; export const registerConversationsRoutes = ( router: ElasticAssistantPluginRouter, @@ -25,6 +26,7 @@ export const registerConversationsRoutes = ( readConversationRoute(router); updateConversationRoute(router); deleteConversationRoute(router); + readLastConversationRoute(router); // Conversations bulk CRUD bulkActionConversationsRoute(router, logger); diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 86156d0ba0907..1d4851386f303 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -76,6 +76,8 @@ export class RequestContextFactory implements IRequestContextFactory { return appContextService.getRegisteredTools(pluginName); }, + telemetry: core.analytics, + getAIAssistantSOClient: memoize(() => { const username = startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 6118d9e517ada..ebe7824250601 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -12,7 +12,7 @@ import { AssistantProvider as ElasticAssistantProvider } from '@kbn/elastic-assi import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; -import { augmentMessageCodeBlocks, LOCAL_STORAGE_KEY } from './helpers'; +import { augmentMessageCodeBlocks } from './helpers'; import { useBaseConversations } from './use_conversation_store'; import { DEFAULT_ALLOW, DEFAULT_ALLOW_REPLACEMENT } from './content/anonymization'; import { PROMPT_CONTEXTS } from './content/prompt_contexts'; @@ -20,7 +20,6 @@ import { BASE_SECURITY_QUICK_PROMPTS } from './content/quick_prompts'; import { BASE_SECURITY_SYSTEM_PROMPTS } from './content/prompts/system'; import { useAnonymizationStore } from './use_anonymization_store'; import { useAssistantAvailability } from './use_assistant_availability'; -import { APP_ID } from '../../common/constants'; import { useAppToasts } from '../common/hooks/use_app_toasts'; import { useIsExperimentalFeatureEnabled } from '../common/hooks/use_experimental_features'; import { useSignalIndex } from '../detections/containers/detection_engine/alerts/use_signal_index'; @@ -49,8 +48,6 @@ export const AssistantProvider: React.FC = ({ children }) => { const { defaultAllow, defaultAllowReplacement, setDefaultAllow, setDefaultAllowReplacement } = useAnonymizationStore(); - const nameSpace = `${APP_ID}.${LOCAL_STORAGE_KEY}`; - const { signalIndexName } = useSignalIndex(); const alertsIndexPattern = signalIndexName ?? undefined; const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) @@ -61,7 +58,7 @@ export const AssistantProvider: React.FC = ({ children }) => { alertsIndexPattern={alertsIndexPattern} augmentMessageCodeBlocks={augmentMessageCodeBlocks} assistantAvailability={assistantAvailability} - assistantTelemetry={assistantTelemetry} // to server + assistantTelemetry={assistantTelemetry} defaultAllow={defaultAllow} // to server and plugin start defaultAllowReplacement={defaultAllowReplacement} // to server and plugin start docLinks={{ ELASTIC_WEBSITE_URL, DOC_LINK_VERSION }} @@ -76,13 +73,8 @@ export const AssistantProvider: React.FC = ({ children }) => { http={http} assistantStreamingEnabled={assistantStreamingEnabled} modelEvaluatorEnabled={isModelEvaluationEnabled} - nameSpace={nameSpace} - ragOnAlerts={ragOnAlerts} setDefaultAllow={setDefaultAllow} // remove setDefaultAllowReplacement={setDefaultAllowReplacement} // remove - setConversations={setConversations} - setDefaultAllow={setDefaultAllow} - setDefaultAllowReplacement={setDefaultAllowReplacement} title={ASSISTANT_TITLE} toasts={toasts} > diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index d47dbe2786a03..76ce02187a742 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { useFetchConversationsByUser, type Conversation } from '@kbn/elastic-assistant'; +import { useFetchCurrentUserConversations, type Conversation } from '@kbn/elastic-assistant'; import { merge, unset } from 'lodash/fp'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; @@ -25,7 +25,7 @@ export const useConversationStore = (): Record => { [isDataQualityDashboardPageExists] ); - const { data: conversationsData, isLoading, refresh } = useFetchConversationsByUser(); + const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); useEffect(() => { if (!isLoading) { @@ -35,18 +35,22 @@ export const useConversationStore = (): Record => { transformed[conversation.id] = conversation; return transformed; }, {}); + + const notUsedBaseConversations = Object.keys(baseConversations).filter( + (baseId) => (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ); + + console.log(notUsedBaseConversations) setConversations( merge( userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { + notUsedBaseConversations.reduce>( + (transformed, conversation) => { transformed[conversation] = baseConversations[conversation]; return transformed; - }, {}) + }, + {} + ) ) ); } From dd501f6b23dbcd35acdb943dc018d24b0dec5004 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 11 Jan 2024 20:32:36 -0800 Subject: [PATCH 005/141] fixed merge issues --- .../kbn-elastic-assistant/impl/assistant_context/index.tsx | 2 -- .../plugins/security_solution/public/assistant/provider.tsx | 6 ------ .../public/assistant/use_conversation_store/index.tsx | 1 - 3 files changed, 9 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 26687fe3cbffe..26686afa77607 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -88,7 +88,6 @@ export interface AssistantProviderProps { }) => EuiCommentProps[]; http: HttpSetup; baseConversations: Record; - getInitialConversations: () => Record; nameSpace?: string; // setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; @@ -176,7 +175,6 @@ export const AssistantProvider: React.FC = ({ getComments, http, baseConversations, - getInitialConversations, nameSpace = DEFAULT_ASSISTANT_NAMESPACE, setDefaultAllow, setDefaultAllowReplacement, diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index c7e00bfc81756..5b5f60f322c8c 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -68,14 +68,8 @@ export const AssistantProvider: React.FC = ({ children }) => { baseConversations={baseConversations} getComments={getComments} http={http} - assistantStreamingEnabled={assistantStreamingEnabled} - modelEvaluatorEnabled={isModelEvaluationEnabled} setDefaultAllow={setDefaultAllow} // remove setDefaultAllowReplacement={setDefaultAllowReplacement} // remove - nameSpace={nameSpace} - setConversations={setConversations} - setDefaultAllow={setDefaultAllow} - setDefaultAllowReplacement={setDefaultAllowReplacement} title={ASSISTANT_TITLE} toasts={toasts} > diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index 76ce02187a742..8bb563599eef6 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -40,7 +40,6 @@ export const useConversationStore = (): Record => { (baseId) => (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined ); - console.log(notUsedBaseConversations) setConversations( merge( userConversations, From 7af9db242881eb00dfacd982259b7d01ffb5d8ff Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 13 Jan 2024 10:07:31 -0800 Subject: [PATCH 006/141] added kbn-data-stream-adapter --- package.json | 1 + packages/kbn-data-stream-adapter/README.md | 71 ++++ packages/kbn-data-stream-adapter/ecs.ts | 9 + packages/kbn-data-stream-adapter/index.ts | 17 + .../kbn-data-stream-adapter/jest.config.js | 13 + packages/kbn-data-stream-adapter/kibana.jsonc | 5 + packages/kbn-data-stream-adapter/package.json | 7 + ...reate_or_update_component_template.test.ts | 377 +++++++++++++++++ .../create_or_update_component_template.ts | 113 +++++ .../src/create_or_update_data_stream.test.ts | 172 ++++++++ .../src/create_or_update_data_stream.ts | 239 +++++++++++ .../create_or_update_index_template.test.ts | 167 ++++++++ .../src/create_or_update_index_template.ts | 66 +++ .../src/data_stream_adapter.ts | 172 ++++++++ .../src/data_stream_spaces_adapter.ts | 115 +++++ .../src/field_maps/ecs_field_map.ts | 88 ++++ .../field_maps/mapping_from_field_map.test.ts | 393 ++++++++++++++++++ .../src/field_maps/mapping_from_field_map.ts | 54 +++ .../src/field_maps/types.ts | 56 +++ .../src/install_with_timeout.test.ts | 63 +++ .../src/install_with_timeout.ts | 67 +++ .../src/resource_installer_utils.test.ts | 170 ++++++++ .../src/resource_installer_utils.ts | 106 +++++ .../src/retry_transient_es_errors.test.ts | 78 ++++ .../src/retry_transient_es_errors.ts | 60 +++ .../kbn-data-stream-adapter/tsconfig.json | 25 ++ tsconfig.base.json | 2 + yarn.lock | 4 + 28 files changed, 2710 insertions(+) create mode 100644 packages/kbn-data-stream-adapter/README.md create mode 100644 packages/kbn-data-stream-adapter/ecs.ts create mode 100644 packages/kbn-data-stream-adapter/index.ts create mode 100644 packages/kbn-data-stream-adapter/jest.config.js create mode 100644 packages/kbn-data-stream-adapter/kibana.jsonc create mode 100644 packages/kbn-data-stream-adapter/package.json create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts create mode 100644 packages/kbn-data-stream-adapter/src/data_stream_adapter.ts create mode 100644 packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts create mode 100644 packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts create mode 100644 packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts create mode 100644 packages/kbn-data-stream-adapter/src/field_maps/types.ts create mode 100644 packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/install_with_timeout.ts create mode 100644 packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/resource_installer_utils.ts create mode 100644 packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts create mode 100644 packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts create mode 100644 packages/kbn-data-stream-adapter/tsconfig.json diff --git a/package.json b/package.json index 73e4302454979..23cf19c06cc03 100644 --- a/package.json +++ b/package.json @@ -367,6 +367,7 @@ "@kbn/data-plugin": "link:src/plugins/data", "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search", "@kbn/data-service": "link:packages/kbn-data-service", + "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter", "@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor", "@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example", "@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor", diff --git a/packages/kbn-data-stream-adapter/README.md b/packages/kbn-data-stream-adapter/README.md new file mode 100644 index 0000000000000..f3d869b3554a6 --- /dev/null +++ b/packages/kbn-data-stream-adapter/README.md @@ -0,0 +1,71 @@ +# @kbn/data-stream-adapter + +Utility library to for Elasticsearch data stream creation + +## DataStreamAdapter + +It creates a single data stream. Example: + +``` +// Instantiate +const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); + +// Define component and index templates +dataStream.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +dataStream.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], + template: { + lifecycle: { + data_retention: '5d', + }, + }, +}); + +// Installs templates and data stream, or updates existing. +await dataStream.install({ logger, esClient, pluginStop$ }); +``` + + +## DataStreamSpacesAdapter + +It creates space aware data streams. Example: + +``` +// Instantiate +const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); + +// Define component and index templates +spacesDataStream.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +spacesDataStream.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], + template: { + lifecycle: { + data_retention: '5d', + }, + }, +}); + +// Installs templates and updates existing data streams. +await spacesDataStream.install({ logger, esClient, pluginStop$ }); + +// After installation we can create space-aware data streams on runtime. +await spacesDataStream.installSpace('space2'); // creates `my-awesome-datastream-space2` if it does not exist +``` diff --git a/packages/kbn-data-stream-adapter/ecs.ts b/packages/kbn-data-stream-adapter/ecs.ts new file mode 100644 index 0000000000000..c8993d7f1b5e7 --- /dev/null +++ b/packages/kbn-data-stream-adapter/ecs.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './src/field_maps/ecs_field_map'; diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts new file mode 100644 index 0000000000000..c9e3bd0950d58 --- /dev/null +++ b/packages/kbn-data-stream-adapter/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { DataStreamAdapter } from './src/data_stream_adapter'; +export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter'; +export type { + DataStreamAdapterParams, + SetComponentTemplateParams, + SetIndexTemplateParams, + InstallParams, +} from './src/data_stream_adapter'; +export * from './src/field_maps/types'; diff --git a/packages/kbn-data-stream-adapter/jest.config.js b/packages/kbn-data-stream-adapter/jest.config.js new file mode 100644 index 0000000000000..48b717249e353 --- /dev/null +++ b/packages/kbn-data-stream-adapter/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-data-stream-adapter'], +}; diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc new file mode 100644 index 0000000000000..99cbb458a8517 --- /dev/null +++ b/packages/kbn-data-stream-adapter/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/data-stream-adapter", + "owner": "@elastic/security-threat-hunting-explore" +} diff --git a/packages/kbn-data-stream-adapter/package.json b/packages/kbn-data-stream-adapter/package.json new file mode 100644 index 0000000000000..3f9118a5916bf --- /dev/null +++ b/packages/kbn-data-stream-adapter/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/data-stream-adapter", + "version": "1.0.0", + "description": "Utility library to for Elasticsearch DataStream creation", + "license": "SSPL-1.0 OR Elastic License 2.0", + "private": true +} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts new file mode 100644 index 0000000000000..a20156163d6b6 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts @@ -0,0 +1,377 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; + +const randomDelayMultiplier = 0.01; +const logger = loggingSystemMock.createLogger(); +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const componentTemplate = { + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: false, + properties: { + foo: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, +}; + +describe('createOrUpdateComponentTemplate', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it(`should call esClient to put component template`, async () => { + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledWith(componentTemplate); + }); + + it(`should retry on transient ES errors`, async () => { + clusterClient.cluster.putComponentTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should log and throw error if max retries exceeded`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue( + new EsErrors.ConnectionError('foo') + ); + await expect(() => + createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template test-mappings - foo` + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + }); + + it(`should log and throw error if ES throws error`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('generic error')); + + await expect(() => + createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template test-mappings - generic error` + ); + }); + + it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { + error: { + root_cause: [ + { + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + }, + ], + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', + caused_by: { + type: 'illegal_argument_exception', + reason: 'Limit of total fields [1900] has been exceeded', + }, + }, + }, + }, + }, + }) + ) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], + }); + + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, + }, + }, + }, + }); + }); + + it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { + error: { + root_cause: [ + { + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + }, + ], + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', + caused_by: { + type: 'illegal_argument_exception', + reason: 'Limit of total fields [1900] has been exceeded', + }, + }, + }, + }, + }, + }) + ) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ + existingIndexTemplate, + { + name: 'lyndon', + // @ts-expect-error + index_template: { + index_patterns: ['intel*'], + }, + }, + { + name: 'sample_ds', + // @ts-expect-error + index_template: { + index_patterns: ['sample_ds-*'], + data_stream: { + hidden: false, + allow_custom_routing: false, + }, + }, + }, + ], + }); + + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, + }, + }, + }, + }); + }); + + it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { + error: { + root_cause: [ + { + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + }, + ], + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', + caused_by: { + type: 'illegal_argument_exception', + reason: 'Limit of total fields [1900] has been exceeded', + }, + }, + }, + }, + }, + }) + ) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + clusterClient.indices.getIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], + }); + clusterClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts new file mode 100644 index 0000000000000..f3c2e55d5569e --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + ClusterPutComponentTemplateRequest, + IndicesGetIndexTemplateIndexTemplateItem, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { asyncForEach } from '@kbn/std'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface CreateOrUpdateComponentTemplateOpts { + logger: Logger; + esClient: ElasticsearchClient; + template: ClusterPutComponentTemplateRequest; + totalFieldsLimit: number; +} + +const putIndexTemplateTotalFieldsLimitUsingComponentTemplate = async ( + esClient: ElasticsearchClient, + componentTemplateName: string, + totalFieldsLimit: number, + logger: Logger +) => { + // Get all index templates and filter down to just the ones referencing this component template + const { index_templates: indexTemplates } = await retryTransientEsErrors( + () => esClient.indices.getIndexTemplate(), + { logger } + ); + const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter( + (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) => + (indexTemplate.index_template?.composed_of ?? []).includes(componentTemplateName) + ); + + await asyncForEach( + indexTemplatesUsingComponentTemplate, + async (template: IndicesGetIndexTemplateIndexTemplateItem) => { + await retryTransientEsErrors( + () => + esClient.indices.putIndexTemplate({ + name: template.name, + body: { + ...template.index_template, + template: { + ...template.index_template.template, + settings: { + ...template.index_template.template?.settings, + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + }, + }, + }), + { logger } + ); + } + ); +}; + +const createOrUpdateComponentTemplateHelper = async ( + esClient: ElasticsearchClient, + template: ClusterPutComponentTemplateRequest, + totalFieldsLimit: number, + logger: Logger +) => { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger }); + } catch (error) { + const reason = error?.meta?.body?.error?.caused_by?.caused_by?.caused_by?.reason; + if (reason && reason.match(/Limit of total fields \[\d+\] has been exceeded/) != null) { + // This error message occurs when there is an index template using this component template + // that contains a field limit setting that using this component template exceeds + // Specifically, this can happen for the ECS component template when we add new fields + // to adhere to the ECS spec. Individual index templates specify field limits so if the + // number of new ECS fields pushes the composed mapping above the limit, this error will + // occur. We have to update the field limit inside the index template now otherwise we + // can never update the component template + await putIndexTemplateTotalFieldsLimitUsingComponentTemplate( + esClient, + template.name, + totalFieldsLimit, + logger + ); + + // Try to update the component template again + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { + logger, + }); + } else { + throw error; + } + } +}; + +export const createOrUpdateComponentTemplate = async ({ + logger, + esClient, + template, + totalFieldsLimit, +}: CreateOrUpdateComponentTemplateOpts) => { + logger.info(`Installing component template ${template.name}`); + + try { + await createOrUpdateComponentTemplateHelper(esClient, template, totalFieldsLimit, logger); + } catch (err) { + logger.error(`Error installing component template ${template.name} - ${err.message}`); + throw err; + } +}; diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts new file mode 100644 index 0000000000000..cc587dcaebfad --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { + updateDataStreams, + createDataStream, + createOrUpdateDataStream, +} from './create_or_update_data_stream'; + +const logger = loggingSystemMock.createLogger(); +const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +esClient.indices.putMapping.mockResolvedValue({ acknowledged: true }); +esClient.indices.putSettings.mockResolvedValue({ acknowledged: true }); + +const simulateIndexTemplateResponse = { template: { mappings: { is_managed: true } } }; +esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse); + +const name = 'test_data_stream'; +const totalFieldsLimit = 1000; + +describe('updateDataStreams', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should update data streams`, async () => { + const dataStreamName = 'test_data_stream-default'; + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: dataStreamName } as IndicesDataStream], + }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: dataStreamName, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name: dataStreamName, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: dataStreamName, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); + + it(`should update multiple data streams`, async () => { + const dataStreamName1 = 'test_data_stream-1'; + const dataStreamName2 = 'test_data_stream-2'; + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: dataStreamName1 }, { name: dataStreamName2 }] as IndicesDataStream[], + }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2); + }); + + it(`should not update data streams when not exist`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).not.toHaveBeenCalled(); + expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(esClient.indices.putMapping).not.toHaveBeenCalled(); + }); +}); + +describe('createDataStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create data stream`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); + }); + + it(`should not create data stream if already exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], + }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).not.toHaveBeenCalled(); + }); +}); + +describe('createOrUpdateDataStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create data stream if not exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); + }); + + it(`should update data stream if already exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], + }); + + await createOrUpdateDataStream({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: name, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: name, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts new file mode 100644 index 0000000000000..5cff6005ea8e0 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { get } from 'lodash'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface UpdateIndexMappingsOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexNames: string[]; + totalFieldsLimit: number; +} + +interface UpdateIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexName: string; + totalFieldsLimit: number; +} + +const updateTotalFieldLimitSetting = async ({ + logger, + esClient, + indexName, + totalFieldsLimit, +}: UpdateIndexOpts) => { + logger.debug(`Updating total field limit setting for ${indexName} data stream.`); + + try { + const body = { 'index.mapping.total_fields.limit': totalFieldsLimit }; + await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), { + logger, + }); + } catch (err) { + logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}` + ); + throw err; + } +}; + +// This will update the mappings but *not* the settings. This +// is due to the fact settings can be classed as dynamic and static, and static +// updates will fail on an index that isn't closed. New settings *will* be applied as part +// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 +const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => { + logger.debug(`Updating mappings for ${indexName} data stream.`); + + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await retryTransientEsErrors( + () => esClient.indices.simulateIndexTemplate({ name: indexName }), + { logger } + ); + } catch (err) { + logger.error( + `Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`); + return; + } + + try { + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }), + { logger } + ); + } catch (err) { + logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`); + throw err; + } +}; +/** + * Updates the data stream mapping and total field limit setting + */ +const updateDataStreamMappings = async ({ + logger, + esClient, + totalFieldsLimit, + indexNames, +}: UpdateIndexMappingsOpts) => { + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + indexNames.map((indexName) => + updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName }) + ) + ); + // Update mappings of the found indices. + await Promise.all( + indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName })) + ); +}; + +export interface CreateOrUpdateDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function createOrUpdateDataStream({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateDataStreamParams): Promise { + logger.info(`Creating data stream - ${name}`); + + // check if data stream exists + let dataStreamExists = false; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreamExists = response.data_streams.length > 0; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + + // if a data stream exists, update the underlying mapping + if (dataStreamExists) { + await updateDataStreamMappings({ + logger, + esClient, + indexNames: [name], + totalFieldsLimit, + }); + } else { + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating data stream ${name} - ${error.message}`); + throw error; + } + } + } +} + +export interface CreateDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; +} + +export async function createDataStream({ + logger, + esClient, + name, +}: CreateDataStreamParams): Promise { + logger.info(`Creating data stream - ${name}`); + + // check if data stream exists + let dataStreamExists = false; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreamExists = response.data_streams.length > 0; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + + // return if data stream already created + if (dataStreamExists) { + return; + } + + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating data stream ${name} - ${error.message}`); + throw error; + } + } +} + +export interface CreateOrUpdateSpacesDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function updateDataStreams({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateSpacesDataStreamParams): Promise { + logger.info(`Updating data streams - ${name}`); + + // check if data stream exists + let dataStreams: IndicesDataStream[] = []; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreams = response.data_streams; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + if (dataStreams.length > 0) { + await updateDataStreamMappings({ + logger, + esClient, + totalFieldsLimit, + indexNames: dataStreams.map((dataStream) => dataStream.name), + }); + } +} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts new file mode 100644 index 0000000000000..cb3b6e77a02b5 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; + +const randomDelayMultiplier = 0.01; +const logger = loggingSystemMock.createLogger(); +const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const getIndexTemplate = (namespace: string = 'default', useDataStream: boolean = false) => ({ + name: `.alerts-test.alerts-${namespace}-index-template`, + body: { + _meta: { + kibana: { + version: '8.6.1', + }, + managed: true, + namespace, + }, + composed_of: ['mappings1', 'framework-mappings'], + index_patterns: [`.internal.alerts-test.alerts-${namespace}-*`], + template: { + mappings: { + _meta: { + kibana: { + version: '8.6.1', + }, + managed: true, + namespace, + }, + dynamic: false, + }, + settings: { + auto_expand_replicas: '0-1', + hidden: true, + ...(useDataStream + ? {} + : { + 'index.lifecycle': { + name: 'test-ilm-policy', + rollover_alias: `.alerts-test.alerts-${namespace}`, + }, + }), + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': 2500, + }, + }, + priority: namespace.length, + }, +}); + +const simulateTemplateResponse = { + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, +}; + +describe('createOrUpdateIndexTemplate', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it(`should call esClient to put index template`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(esClient.indices.simulateTemplate).toHaveBeenCalledWith(getIndexTemplate()); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith(getIndexTemplate()); + }); + + it(`should retry on transient ES errors`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should retry simulateTemplate on transient ES errors`, async () => { + esClient.indices.simulateTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockResolvedValue({ acknowledged: true }); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate }); + + expect(esClient.indices.simulateTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should log and throw error if max retries exceeded`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockRejectedValue(new EsErrors.ConnectionError('foo')); + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test.alerts-default-index-template - foo`, + expect.any(Error) + ); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(4); + }); + + it(`should log and throw error if ES throws error`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); + + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test.alerts-default-index-template - generic error`, + expect.any(Error) + ); + }); + + it(`should log and return without updating template if simulate throws error`, async () => { + esClient.indices.simulateTemplate.mockRejectedValue(new Error('simulate error')); + esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); + + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(logger.error).toHaveBeenCalledWith( + `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - simulate error`, + expect.any(Error) + ); + expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + }); + + it(`should throw error if simulate returns empty mappings`, async () => { + esClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...simulateTemplateResponse, + template: { + ...simulateTemplateResponse.template, + mappings: {}, + }, + })); + + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping"` + ); + expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts new file mode 100644 index 0000000000000..c80abf3c8045e --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndicesPutIndexTemplateRequest, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { isEmpty } from 'lodash'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface CreateOrUpdateIndexTemplateOpts { + logger: Logger; + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; +} + +/** + * Installs index template that uses installed component template + * Prior to installation, simulates the installation to check for possible + * conflicts. Simulate should return an empty mapping if a template + * conflicts with an already installed template. + */ +export const createOrUpdateIndexTemplate = async ({ + logger, + esClient, + template, +}: CreateOrUpdateIndexTemplateOpts) => { + logger.info(`Installing index template ${template.name}`); + + let mappings: MappingTypeMapping = {}; + try { + // Simulate the index template to proactively identify any issues with the mapping + const simulateResponse = await retryTransientEsErrors( + () => esClient.indices.simulateTemplate(template), + { logger } + ); + mappings = simulateResponse.template.mappings; + } catch (err) { + logger.error( + `Failed to simulate index template mappings for ${template.name}; not applying mappings - ${err.message}`, + err + ); + return; + } + + if (isEmpty(mappings)) { + throw new Error( + `No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping` + ); + } + + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { + logger, + }); + } catch (err) { + logger.error(`Error installing index template ${template.name} - ${err.message}`, err); + throw err; + } +}; diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts new file mode 100644 index 0000000000000..c384a92d90a3a --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + ClusterPutComponentTemplateRequest, + IndicesIndexSettings, + IndicesPutIndexTemplateIndexTemplateMapping, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import type { Subject } from 'rxjs'; +import type { FieldMap } from './field_maps/types'; +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { createOrUpdateDataStream } from './create_or_update_data_stream'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; +import { InstallShutdownError, installWithTimeout } from './install_with_timeout'; +import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils'; + +export interface DataStreamAdapterParams { + kibanaVersion: string; + totalFieldsLimit?: number; +} +export interface SetComponentTemplateParams { + name: string; + fieldMap: FieldMap; + settings?: IndicesIndexSettings; + dynamic?: 'strict' | boolean; +} +export interface SetIndexTemplateParams { + name: string; + componentTemplateRefs?: string[]; + namespace?: string; + template?: IndicesPutIndexTemplateIndexTemplateMapping; + hidden?: boolean; +} + +export interface GetInstallFnParams { + logger: Logger; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} +export interface InstallParams { + logger: Logger; + esClientPromise: Promise; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} + +export interface InstallationPromise { + result: boolean; + error?: string; +} + +export const successResult = () => ({ result: true }); +export const errorResult = (error?: string) => ({ result: false, error }); + +const DEFAULT_FIELDS_LIMIT = 2500; + +export class DataStreamAdapter { + protected readonly kibanaVersion: string; + protected readonly totalFieldsLimit: number; + protected componentTemplates: ClusterPutComponentTemplateRequest[] = []; + protected indexTemplates: IndicesPutIndexTemplateRequest[] = []; + protected installed: boolean; + + constructor(protected readonly name: string, options: DataStreamAdapterParams) { + this.installed = false; + this.kibanaVersion = options.kibanaVersion; + this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT; + } + + public setComponentTemplate(params: SetComponentTemplateParams) { + if (this.installed) { + throw new Error('Cannot set component template after install'); + } + this.componentTemplates.push(getComponentTemplate(params)); + } + + public setIndexTemplate(params: SetIndexTemplateParams) { + if (this.installed) { + throw new Error('Cannot set index template after install'); + } + this.indexTemplates.push( + getIndexTemplate({ + ...params, + indexPatterns: [this.name], + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: this.totalFieldsLimit, + }) + ); + } + + protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) { + return async (promise: Promise, description?: string): Promise => { + try { + await installWithTimeout({ + installFn: () => promise, + description, + timeoutMs: tasksTimeoutMs, + pluginStop$, + }); + } catch (err) { + if (err instanceof InstallShutdownError) { + logger.info(err.message); + } else { + throw err; + } + } + }; + } + + public async install({ logger, esClientPromise, pluginStop$, tasksTimeoutMs }: InstallParams) { + if (this.installed) { + throw new Error('Cannot re-install data stream'); + } + try { + const esClient = await esClientPromise; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${componentTemplate.name} component template` + ) + ) + ); + + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ + template: indexTemplate, + esClient, + logger, + }), + `${indexTemplate.name} index template` + ) + ) + ); + + // create data stream when everything is ready + await installFn( + createOrUpdateDataStream({ + name: this.name, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${this.name} data stream` + ); + this.installed = true; + return successResult(); + } catch (error) { + logger.error(`Error initializing data stream resources: ${error.message}`); + this.installed = false; + return errorResult(error.message); + } + } +} diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts new file mode 100644 index 0000000000000..f2568239e77c9 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { createDataStream, updateDataStreams } from './create_or_update_data_stream'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; +import { + DataStreamAdapter, + InstallationPromise, + type DataStreamAdapterParams, + type InstallParams, + successResult, + errorResult, +} from './data_stream_adapter'; + +export class DataStreamSpacesAdapter extends DataStreamAdapter { + private installedSpaceDataStreamName: Map>; + private _installSpace?: (spaceId: string) => Promise; + + constructor(private readonly prefix: string, options: DataStreamAdapterParams) { + super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names + this.installedSpaceDataStreamName = new Map(); + } + + public async install({ + logger, + esClientPromise, + pluginStop$, + tasksTimeoutMs, + }: InstallParams): Promise { + if (this.installed) { + throw new Error('Cannot re-install data stream'); + } + + try { + const esClient = await esClientPromise; + + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `create or update ${componentTemplate.name} component template` + ) + ) + ); + + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), + `create or update ${indexTemplate.name} index template` + ) + ) + ); + + // Update existing space data streams + await installFn( + updateDataStreams({ + name: `${this.prefix}-*`, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `update space data streams` + ); + + // define function to install data stream for spaces on demand + this._installSpace = async (spaceId: string) => { + const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); + if (existingInstallPromise) { + return existingInstallPromise; + } + const name = `${this.prefix}-${spaceId}`; + const installPromise = installFn( + createDataStream({ name, esClient, logger }), + `create ${name} data stream` + ).then(() => name); + + this.installedSpaceDataStreamName.set(spaceId, installPromise); + return installPromise; + }; + + this.installed = true; + return successResult(); + } catch (error) { + logger.error(`Error initializing data stream resources: ${error.message}`); + this.installed = false; + return errorResult(error.message); + } + } + + public async installSpace(spaceId: string): Promise { + if (!this._installSpace) { + throw new Error('Cannot installSpace before install'); + } + return this._installSpace(spaceId); + } + + public async getSpaceIndexName(spaceId: string): Promise { + return this.installedSpaceDataStreamName.get(spaceId); + } +} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts new file mode 100644 index 0000000000000..17e8af1da7887 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EcsFlat } from '@kbn/ecs'; +import type { EcsMetadata, FieldMap } from './types'; + +const EXCLUDED_TYPES = ['constant_keyword']; + +// ECS fields that have reached Stage 2 in the RFC process +// are included in the generated Yaml but are still considered +// experimental. Some are correctly marked as beta but most are +// not. + +// More about the RFC stages here: https://elastic.github.io/ecs/stages.html + +// The following RFCS are currently in stage 2: +// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md + +// Fields from these RFCs that are not already in the ECS component template +// as of 8.11 are manually identified as experimental below. +// The next time this list is updated, we should check the above list of RFCs to +// see if any have moved to Stage 3 and remove them from the list and check if +// there are any new stage 2 RFCs with fields we should exclude as experimental. + +const EXPERIMENTAL_FIELDS = [ + 'faas.trigger', // this was previously mapped as nested but changed to object + 'faas.trigger.request_id', + 'faas.trigger.type', + 'host.cpu.system.norm.pct', + 'host.cpu.user.norm.pct', + 'host.fsstats.total_size.total', + 'host.fsstats.total_size.used', + 'host.fsstats.total_size.used.pct', + 'host.load.norm.1', + 'host.load.norm.5', + 'host.load.norm.15', + 'host.memory.actual.used.bytes', + 'host.memory.actual.used.pct', + 'host.memory.total', + 'process.io.bytes', + 'volume.bus_type', + 'volume.default_access', + 'volume.device_name', + 'volume.device_type', + 'volume.dos_name', + 'volume.file_system_type', + 'volume.mount_name', + 'volume.nt_name', + 'volume.product_id', + 'volume.product_name', + 'volume.removable', + 'volume.serial_number', + 'volume.size', + 'volume.vendor_id', + 'volume.vendor_name', + 'volume.writable', +]; + +export const ecsFieldMap: FieldMap = Object.fromEntries( + Object.entries(EcsFlat) + .filter( + ([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key) + ) + .map(([key, _]) => { + const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat]; + return [ + key, + { + type: value.type, + array: value.normalize.includes('array'), + required: !!value.required, + ...(value.scaling_factor ? { scaling_factor: value.scaling_factor } : {}), + ...(value.ignore_above ? { ignore_above: value.ignore_above } : {}), + ...(value.multi_fields ? { multi_fields: value.multi_fields } : {}), + }, + ]; + }) +); + +export type EcsFieldMap = typeof ecsFieldMap; diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts new file mode 100644 index 0000000000000..e851bdc21d01b --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; +import { mappingFromFieldMap } from './mapping_from_field_map'; + +export const testFieldMap: FieldMap = { + date_field: { + type: 'date', + array: false, + required: true, + }, + keyword_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + long_field: { + type: 'long', + array: false, + required: false, + }, + multifield_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + multi_fields: [ + { + flat_name: 'multifield_field.text', + name: 'text', + type: 'match_only_text', + }, + ], + }, + geopoint_field: { + type: 'geo_point', + array: false, + required: false, + }, + ip_field: { + type: 'ip', + array: false, + required: false, + }, + array_field: { + type: 'keyword', + array: true, + required: false, + ignore_above: 1024, + }, + nested_array_field: { + type: 'nested', + array: false, + required: false, + }, + 'nested_array_field.field1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'nested_array_field.field2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + scaled_float_field: { + type: 'scaled_float', + array: false, + required: false, + scaling_factor: 1000, + }, + constant_keyword_field: { + type: 'constant_keyword', + array: false, + required: false, + }, + 'parent_field.child1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'parent_field.child2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + unmapped_object: { + type: 'object', + required: false, + enabled: false, + }, + formatted_field: { + type: 'date_range', + required: false, + format: 'epoch_millis||strict_date_optional_time', + }, +}; +export const expectedTestMapping = { + properties: { + array_field: { + ignore_above: 1024, + type: 'keyword', + }, + constant_keyword_field: { + type: 'constant_keyword', + }, + date_field: { + type: 'date', + }, + multifield_field: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + geopoint_field: { + type: 'geo_point', + }, + ip_field: { + type: 'ip', + }, + keyword_field: { + ignore_above: 1024, + type: 'keyword', + }, + long_field: { + type: 'long', + }, + nested_array_field: { + properties: { + field1: { + ignore_above: 1024, + type: 'keyword', + }, + field2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + parent_field: { + properties: { + child1: { + ignore_above: 1024, + type: 'keyword', + }, + child2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + scaled_float_field: { + scaling_factor: 1000, + type: 'scaled_float', + }, + unmapped_object: { + enabled: false, + type: 'object', + }, + formatted_field: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + }, +}; + +describe('mappingFromFieldMap', () => { + it('correctly creates mapping from field map', () => { + expect(mappingFromFieldMap(testFieldMap)).toEqual({ + dynamic: 'strict', + ...expectedTestMapping, + }); + expect(mappingFromFieldMap(alertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + '@timestamp': { + ignore_malformed: false, + type: 'date', + }, + event: { + properties: { + action: { + type: 'keyword', + }, + kind: { + type: 'keyword', + }, + }, + }, + kibana: { + properties: { + alert: { + properties: { + action_group: { + type: 'keyword', + }, + case_ids: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + end: { + type: 'date', + }, + flapping: { + type: 'boolean', + }, + flapping_history: { + type: 'boolean', + }, + maintenance_window_ids: { + type: 'keyword', + }, + instance: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + last_detected: { + type: 'date', + }, + reason: { + fields: { + text: { + type: 'match_only_text', + }, + }, + type: 'keyword', + }, + rule: { + properties: { + category: { + type: 'keyword', + }, + consumer: { + type: 'keyword', + }, + execution: { + properties: { + uuid: { + type: 'keyword', + }, + }, + }, + name: { + type: 'keyword', + }, + parameters: { + type: 'flattened', + ignore_above: 4096, + }, + producer: { + type: 'keyword', + }, + revision: { + type: 'long', + }, + rule_type_id: { + type: 'keyword', + }, + tags: { + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + }, + }, + start: { + type: 'date', + }, + status: { + type: 'keyword', + }, + time_range: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + url: { + ignore_above: 2048, + index: false, + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + workflow_assignee_ids: { + type: 'keyword', + }, + workflow_status: { + type: 'keyword', + }, + workflow_tags: { + type: 'keyword', + }, + }, + }, + space_ids: { + type: 'keyword', + }, + version: { + type: 'version', + }, + }, + }, + tags: { + type: 'keyword', + }, + }, + }); + expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + kibana: { + properties: { + alert: { + properties: { + risk_score: { type: 'float' }, + rule: { + properties: { + author: { type: 'keyword' }, + created_at: { type: 'date' }, + created_by: { type: 'keyword' }, + description: { type: 'keyword' }, + enabled: { type: 'keyword' }, + from: { type: 'keyword' }, + interval: { type: 'keyword' }, + license: { type: 'keyword' }, + note: { type: 'keyword' }, + references: { type: 'keyword' }, + rule_id: { type: 'keyword' }, + rule_name_override: { type: 'keyword' }, + to: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + updated_by: { type: 'keyword' }, + version: { type: 'keyword' }, + }, + }, + severity: { type: 'keyword' }, + suppression: { + properties: { + docs_count: { type: 'long' }, + end: { type: 'date' }, + terms: { + properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, + }, + start: { type: 'date' }, + }, + }, + system_status: { type: 'keyword' }, + workflow_reason: { type: 'keyword' }, + workflow_status_updated_at: { type: 'date' }, + workflow_user: { type: 'keyword' }, + }, + }, + }, + }, + ecs: { properties: { version: { type: 'keyword' } } }, + }, + }); + }); + + it('uses dynamic setting if specified', () => { + expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ + dynamic: true, + ...expectedTestMapping, + }); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts new file mode 100644 index 0000000000000..5878cedd44195 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { set } from '@kbn/safer-lodash-set'; +import type { FieldMap, MultiField } from './types'; + +export function mappingFromFieldMap( + fieldMap: FieldMap, + dynamic: 'strict' | boolean = 'strict' +): MappingTypeMapping { + const mappings = { + dynamic, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key: string) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { name, required, array, multi_fields, ...rest } = field; + const mapped = multi_fields + ? { + ...rest, + // eslint-disable-next-line @typescript-eslint/naming-convention + fields: multi_fields.reduce((acc, multi_field: MultiField) => { + acc[multi_field.name] = { + type: multi_field.type, + }; + return acc; + }, {} as Record), + } + : rest; + + set(mappings.properties, field.name.split('.').join('.properties.'), mapped); + + if (name === '@timestamp') { + set(mappings.properties, `${name}.ignore_malformed`, false); + } + }); + + return mappings; +} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/types.ts b/packages/kbn-data-stream-adapter/src/field_maps/types.ts new file mode 100644 index 0000000000000..0a0b68a2f26e6 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/types.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface AllowedValue { + description?: string; + name?: string; +} + +export interface MultiField { + flat_name: string; + name: string; + type: string; +} + +export interface EcsMetadata { + allowed_values?: AllowedValue[]; + dashed_name: string; + description: string; + doc_values?: boolean; + example?: string | number | boolean; + flat_name: string; + ignore_above?: number; + index?: boolean; + level: string; + multi_fields?: MultiField[]; + name: string; + normalize: string[]; + required?: boolean; + scaling_factor?: number; + short: string; + type: string; + properties?: Record; +} + +export interface FieldMap { + [key: string]: { + type: string; + required: boolean; + array?: boolean; + doc_values?: boolean; + enabled?: boolean; + format?: string; + ignore_above?: number; + multi_fields?: MultiField[]; + index?: boolean; + path?: string; + scaling_factor?: number; + dynamic?: boolean | 'strict'; + properties?: Record; + }; +} diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts new file mode 100644 index 0000000000000..59945b23124c6 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggerMock } from '@kbn/logging-mocks'; + +import { installWithTimeout } from './install_with_timeout'; +import { ReplaySubject, type Subject } from 'rxjs'; + +const logger = loggerMock.create(); + +describe('installWithTimeout', () => { + let pluginStop$: Subject; + + beforeEach(() => { + jest.resetAllMocks(); + pluginStop$ = new ReplaySubject(1); + }); + + it(`should call installFn`, async () => { + const installFn = jest.fn(); + await installWithTimeout({ + installFn, + pluginStop$, + timeoutMs: 10, + }); + expect(installFn).toHaveBeenCalled(); + }); + + it(`should short-circuit installFn if it exceeds configured timeout`, async () => { + await expect(() => + installWithTimeout({ + installFn: async () => { + await new Promise((r) => setTimeout(r, 20)); + }, + pluginStop$, + timeoutMs: 10, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failure during installation. Timeout: it took more than 10ms"` + ); + }); + + it(`should short-circuit installFn if pluginStop$ signal is received`, async () => { + pluginStop$.next(); + await expect(() => + installWithTimeout({ + installFn: async () => { + await new Promise((r) => setTimeout(r, 5)); + logger.info(`running`); + }, + pluginStop$, + timeoutMs: 10, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Server is stopping; must stop all async operations"` + ); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.ts new file mode 100644 index 0000000000000..7995fed5152ad --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/install_with_timeout.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { firstValueFrom, type Observable } from 'rxjs'; + +const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes + +interface InstallWithTimeoutOpts { + description?: string; + installFn: () => Promise; + pluginStop$: Observable; + timeoutMs?: number; +} + +export class InstallShutdownError extends Error { + constructor() { + super('Server is stopping; must stop all async operations'); + Object.setPrototypeOf(this, InstallShutdownError.prototype); + } +} + +export const installWithTimeout = async ({ + description, + installFn, + pluginStop$, + timeoutMs = INSTALLATION_TIMEOUT, +}: InstallWithTimeoutOpts): Promise => { + try { + let timeoutId: NodeJS.Timeout; + const install = async (): Promise => { + await installFn(); + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + + const throwTimeoutException = (): Promise => { + return new Promise((_, reject) => { + timeoutId = setTimeout(() => { + const msg = `Timeout: it took more than ${timeoutMs}ms`; + reject(new Error(msg)); + }, timeoutMs); + + firstValueFrom(pluginStop$).then(() => { + clearTimeout(timeoutId); + reject(new InstallShutdownError()); + }); + }); + }; + + await Promise.race([install(), throwTimeoutException()]); + } catch (e) { + if (e instanceof InstallShutdownError) { + throw e; + } else { + const reason = e?.message || 'Unknown reason'; + throw new Error( + `Failure during installation${description ? ` of ${description}` : ''}. ${reason}` + ); + } + } +}; diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts new file mode 100644 index 0000000000000..e53eb7704a06a --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getIndexTemplate, getComponentTemplate } from './resource_installer_utils'; + +describe('getIndexTemplate', () => { + const defaultParams = { + name: 'indexTemplateName', + kibanaVersion: '8.12.1', + indexPatterns: ['indexPattern1', 'indexPattern2'], + componentTemplateRefs: ['template1', 'template2'], + totalFieldsLimit: 2500, + }; + + it('should create index template with given parameters and defaults', () => { + const indexTemplate = getIndexTemplate(defaultParams); + + expect(indexTemplate).toEqual({ + name: defaultParams.name, + body: { + data_stream: { hidden: true }, + index_patterns: defaultParams.indexPatterns, + composed_of: defaultParams.componentTemplateRefs, + template: { + settings: { + hidden: true, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + }, + }, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + priority: 7, + }, + }); + }); + + it('should create not hidden index template', () => { + const { body } = getIndexTemplate({ ...defaultParams, hidden: false }); + expect(body?.data_stream?.hidden).toEqual(false); + expect(body?.template?.settings?.hidden).toEqual(false); + }); + + it('should create index template with custom namespace', () => { + const { body } = getIndexTemplate({ ...defaultParams, namespace: 'custom-namespace' }); + expect(body?._meta?.namespace).toEqual('custom-namespace'); + expect(body?.priority).toEqual(16); + }); + + it('should create index template with template overrides', () => { + const { body } = getIndexTemplate({ + ...defaultParams, + template: { + settings: { + number_of_shards: 1, + }, + mappings: { + dynamic: true, + }, + lifecycle: { + data_retention: '30d', + }, + }, + }); + + expect(body?.template?.settings).toEqual({ + hidden: true, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, + number_of_shards: 1, + }); + + expect(body?.template?.mappings).toEqual({ + dynamic: true, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + }); + + expect(body?.template?.lifecycle).toEqual({ + data_retention: '30d', + }); + }); +}); + +describe('getComponentTemplate', () => { + const defaultParams = { + name: 'componentTemplateName', + kibanaVersion: '8.12.1', + fieldMap: { + field1: { type: 'text', required: true }, + field2: { type: 'keyword', required: false }, + }, + }; + + it('should create component template with given parameters and defaults', () => { + const componentTemplate = getComponentTemplate(defaultParams); + + expect(componentTemplate).toEqual({ + name: defaultParams.name, + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: 'strict', + properties: { + field1: { + type: 'text', + }, + field2: { + type: 'keyword', + }, + }, + }, + }, + }); + }); + + it('should create component template with custom settings', () => { + const { template } = getComponentTemplate({ + ...defaultParams, + settings: { + number_of_shards: 1, + number_of_replicas: 1, + }, + }); + + expect(template.settings).toEqual({ + number_of_shards: 1, + number_of_replicas: 1, + 'index.mapping.total_fields.limit': 1500, + }); + }); + + it('should create component template with custom dynamic', () => { + const { template } = getComponentTemplate({ ...defaultParams, dynamic: true }); + expect(template.mappings?.dynamic).toEqual(true); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts new file mode 100644 index 0000000000000..456be9ad8e86f --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndicesPutIndexTemplateRequest, + Metadata, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + ClusterPutComponentTemplateRequest, + IndicesIndexSettings, + IndicesPutIndexTemplateIndexTemplateMapping, +} from '@elastic/elasticsearch/lib/api/types'; +import type { FieldMap } from './field_maps/types'; +import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; + +interface GetComponentTemplateOpts { + name: string; + fieldMap: FieldMap; + settings?: IndicesIndexSettings; + dynamic?: 'strict' | boolean; +} + +export const getComponentTemplate = ({ + name, + fieldMap, + settings, + dynamic = 'strict', +}: GetComponentTemplateOpts): ClusterPutComponentTemplateRequest => ({ + name, + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': + Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, + ...settings, + }, + mappings: mappingFromFieldMap(fieldMap, dynamic), + }, +}); + +interface GetIndexTemplateOpts { + name: string; + indexPatterns: string[]; + kibanaVersion: string; + totalFieldsLimit: number; + componentTemplateRefs?: string[]; + namespace?: string; + template?: IndicesPutIndexTemplateIndexTemplateMapping; + hidden?: boolean; +} + +export const getIndexTemplate = ({ + name, + indexPatterns, + kibanaVersion, + totalFieldsLimit, + componentTemplateRefs, + namespace = 'default', + template = {}, + hidden = true, +}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => { + const indexMetadata: Metadata = { + kibana: { + version: kibanaVersion, + }, + managed: true, + namespace, + }; + + return { + name, + body: { + data_stream: { hidden }, + index_patterns: indexPatterns, + composed_of: componentTemplateRefs, + template: { + ...template, + settings: { + hidden, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': totalFieldsLimit, + ...template.settings, + }, + mappings: { + dynamic: false, + _meta: indexMetadata, + ...template.mappings, + }, + }, + _meta: indexMetadata, + + // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace + // then newly created indices will use the matching template with the *longest* namespace + priority: namespace.length, + }, + }; +}; diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts new file mode 100644 index 0000000000000..f7d6cca8c5a07 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors, type DiagnosticResult } from '@elastic/elasticsearch'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +const mockLogger = loggingSystemMock.createLogger(); + +// mock setTimeout to avoid waiting in tests and prevent test flakiness +global.setTimeout = jest.fn((cb) => jest.fn(cb())) as unknown as typeof global.setTimeout; + +describe('retryTransientEsErrors', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + { error: new EsErrors.ConnectionError('test error'), errorType: 'ConnectionError' }, + { + error: new EsErrors.NoLivingConnectionsError('test error', {} as DiagnosticResult), + errorType: 'NoLivingConnectionsError', + }, + { error: new EsErrors.TimeoutError('test error'), errorType: 'TimeoutError' }, + { + error: new EsErrors.ResponseError({ statusCode: 503 } as DiagnosticResult), + errorType: 'ResponseError (Unavailable)', + }, + { + error: new EsErrors.ResponseError({ statusCode: 408 } as DiagnosticResult), + errorType: 'ResponseError (RequestTimeout)', + }, + { + error: new EsErrors.ResponseError({ statusCode: 410 } as DiagnosticResult), + errorType: 'ResponseError (Gone)', + }, + ])('should retry $errorType', async ({ error }) => { + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + mockFn.mockResolvedValueOnce('success'); + + const result = await retryTransientEsErrors(mockFn, { logger: mockLogger }); + + expect(result).toEqual('success'); + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.error).not.toHaveBeenCalled(); + }); + + it('should throw non-transient errors', async () => { + const error = new EsErrors.ResponseError({ statusCode: 429 } as DiagnosticResult); + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + + await expect(retryTransientEsErrors(mockFn, { logger: mockLogger })).rejects.toEqual(error); + + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).not.toHaveBeenCalled(); + }); + + it('should throw if max retries exceeded', async () => { + const error = new EsErrors.ConnectionError('test error'); + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + mockFn.mockRejectedValueOnce(error); + + await expect( + retryTransientEsErrors(mockFn, { logger: mockLogger, attempt: 2 }) + ).rejects.toEqual(error); + + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts new file mode 100644 index 0000000000000..893c477223ca3 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Logger } from '@kbn/core/server'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +const MAX_ATTEMPTS = 3; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: Error) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && + e?.statusCode && + retryResponseStatuses.includes(e.statusCode)); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { + logger, + attempt = 0, + }: { + logger: Logger; + attempt?: number; + } +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... + + logger.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + // delay with some randomness + await delay(retryDelaySec * 1000 * Math.random()); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json new file mode 100644 index 0000000000000..4fdd3065124fb --- /dev/null +++ b/packages/kbn-data-stream-adapter/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react", + "@emotion/react/types/css-prop", + "@testing-library/jest-dom", + "@testing-library/react" + ] + }, + "include": ["**/*.ts", "**/*.tsx"], + "kbn_references": [ + "@kbn/core", + "@kbn/core-elasticsearch-client-server-mocks", + "@kbn/std", + "@kbn/ecs", + "@kbn/alerts-as-data-utils", + "@kbn/safer-lodash-set", + "@kbn/logging-mocks", + ], + "exclude": ["target/**/*"] +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 406b2f3dda838..6e02dbbea1c36 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -630,6 +630,8 @@ "@kbn/data-search-plugin/*": ["test/plugin_functional/plugins/data_search/*"], "@kbn/data-service": ["packages/kbn-data-service"], "@kbn/data-service/*": ["packages/kbn-data-service/*"], + "@kbn/data-stream-adapter": ["packages/kbn-data-stream-adapter"], + "@kbn/data-stream-adapter/*": ["packages/kbn-data-stream-adapter/*"], "@kbn/data-view-editor-plugin": ["src/plugins/data_view_editor"], "@kbn/data-view-editor-plugin/*": ["src/plugins/data_view_editor/*"], "@kbn/data-view-field-editor-example-plugin": ["examples/data_view_field_editor_example"], diff --git a/yarn.lock b/yarn.lock index ad3e54a1e5efc..bef1ff05cddb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4320,6 +4320,10 @@ version "0.0.0" uid "" +"@kbn/data-stream-adapter@link:packages/kbn-data-stream-adapter": + version "0.0.0" + uid "" + "@kbn/data-view-editor-plugin@link:src/plugins/data_view_editor": version "0.0.0" uid "" From 5d3e92e80d7f30270934574135c2387aded56058 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 13 Jan 2024 19:52:50 -0800 Subject: [PATCH 007/141] improved APIs --- .../src/data_stream_adapter.ts | 5 +- .../src/data_stream_spaces_adapter.ts | 4 +- .../kbn-elastic-assistant-common/constants.ts | 25 + .../kbn-elastic-assistant-common/index.ts | 2 + .../impl/assistant/api/conversations.ts | 25 +- .../use_fetch_current_user_conversations.ts | 38 +- .../impl/assistant/index.tsx | 2 +- .../elastic_assistant/common/constants.ts | 13 - .../server/__mocks__/request.ts | 10 +- .../create_resource_installation_helper.ts | 2 +- .../component_template_from_field_map.test.ts | 79 -- .../component_template_from_field_map.ts | 43 -- .../field_maps/mapping_from_field_map.test.ts | 382 ---------- .../field_maps/mapping_from_field_map.ts | 53 -- .../server/ai_assistant_service/index.ts | 277 +++---- .../lib/conversation_configuration_type.ts | 30 +- .../lib/create_concrete_write_index.test.ts | 718 ------------------ .../lib/create_concrete_write_index.ts | 164 ---- .../lib/create_datastream.mock.ts | 17 - .../lib/create_datastream.ts | 89 --- .../ai_assistant_service/lib/indices.ts | 15 - .../lib/retry_transient_es_errors.ts | 58 -- .../conversations_data_writer.ts | 4 +- .../get_last_conversation.ts | 8 +- .../server/conversations_data_client/index.ts | 33 +- .../elastic_assistant/server/plugin.ts | 55 +- .../routes/conversation/bulk_actions_route.ts | 6 +- .../routes/conversation/create_route.ts | 4 +- .../routes/conversation/delete_route.ts | 15 +- .../server/routes/conversation/find_route.ts | 4 +- .../find_user_conversations_route.ts | 4 +- .../routes/conversation/read_last_route.ts | 8 +- .../server/routes/conversation/read_route.ts | 4 +- .../routes/conversation/update_route.ts | 4 +- .../routes/evaluate/post_evaluate.test.ts | 10 +- .../server/routes/evaluate/post_evaluate.ts | 11 +- .../server/routes/evaluate/utils.ts | 2 +- .../routes/post_actions_connector_execute.ts | 214 +++--- .../server/routes/prompts/create_route.ts | 62 ++ .../server/routes/prompts/delete_route.ts | 70 ++ .../server/routes/prompts/find_route.ts | 68 ++ .../server/routes/prompts/update_route.ts | 76 ++ .../server/routes/register_routes.ts | 48 +- .../server/routes/request_context_factory.ts | 10 +- .../ai_assistant_default_prompts.ts | 61 -- ...t.ts => ai_assistant_prompts_so_client.ts} | 136 ++-- .../assistant_prompts_so_schema.ts | 79 -- .../elastic_assistant_prompts_type.ts | 103 ++- .../post_actions_connector_execute.ts | 48 -- ...ost_actions_connector_execute_route.gen.ts | 81 ++ ...ctions_connector_execute_route.schema.yaml | 118 +++ .../bulk_crud_conversations_route.gen.ts | 10 - .../bulk_crud_conversations_route.schema.yaml | 20 +- .../conversations/common_attributes.gen.ts | 2 +- .../common_attributes.schema.yaml | 2 +- .../crud_conversation_route.gen.ts | 67 ++ ...ml => crud_conversation_route.schema.yaml} | 60 +- .../find_conversations_route.gen.ts | 39 + .../find_conversations_route.schema.yaml | 106 ++- .../server/schemas/evaluate/post_evaluate.ts | 58 -- .../evaluate/post_evaluate_route.gen.ts | 50 ++ .../evaluate/post_evaluate_route.schema.yaml | 108 +++ .../evaluate/post_evaluate_route.schema.yml | 97 --- .../knowledge_base/crud_kb_route.gen.ts | 111 +++ .../knowledge_base/crud_kb_route.schema.yaml | 163 ++++ .../knowledge_base/crud_kb_route.schema.yml | 97 --- .../schemas/prompts/crud_prompts_route.gen.ts | 166 ++++ .../prompts/crud_prompts_route.schema.yaml | 240 ++++++ .../schemas/prompts/find_prompts_route.gen.ts | 60 ++ .../prompts/find_prompts_route.schema.yaml | 108 +++ .../plugins/elastic_assistant/server/types.ts | 9 +- .../plugins/elastic_assistant/tsconfig.json | 1 + 72 files changed, 2276 insertions(+), 2695 deletions(-) create mode 100755 x-pack/packages/kbn-elastic-assistant-common/constants.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts rename x-pack/plugins/elastic_assistant/server/saved_object/{ai_assistant_so_client.ts => ai_assistant_prompts_so_client.ts} (66%) delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts rename x-pack/plugins/elastic_assistant/server/schemas/conversations/{crud_conversation_route.schema.yml => crud_conversation_route.schema.yaml} (64%) delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts index c384a92d90a3a..d2f824971c14b 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts @@ -46,7 +46,7 @@ export interface GetInstallFnParams { } export interface InstallParams { logger: Logger; - esClientPromise: Promise; + esClient: ElasticsearchClient; pluginStop$: Subject; tasksTimeoutMs?: number; } @@ -114,12 +114,11 @@ export class DataStreamAdapter { }; } - public async install({ logger, esClientPromise, pluginStop$, tasksTimeoutMs }: InstallParams) { + public async install({ logger, esClient, pluginStop$, tasksTimeoutMs }: InstallParams) { if (this.installed) { throw new Error('Cannot re-install data stream'); } try { - const esClient = await esClientPromise; const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); // Install component templates in parallel diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts index f2568239e77c9..95c497d62e54a 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts @@ -28,7 +28,7 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter { public async install({ logger, - esClientPromise, + esClient, pluginStop$, tasksTimeoutMs, }: InstallParams): Promise { @@ -37,8 +37,6 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter { } try { - const esClient = await esClientPromise; - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); // Install component templates in parallel diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts new file mode 100755 index 0000000000000..7b75a303ab0e9 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; +export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; + +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; +export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/current_user`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; +export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_last`; + +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_CREATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_create`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_DELETE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_delete`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_UPDATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_update`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_find`; + +export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/prompts`; +export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{promptId}`; +export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_find`; diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts index c64b02160d6e4..103b1d0127424 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts @@ -18,3 +18,5 @@ export { } from './impl/data_anonymization/helpers'; export { transformRawData } from './impl/data_anonymization/transform_raw_data'; + +export * from './constants'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts index 7f2b374789592..c8c67024aa671 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts @@ -8,6 +8,11 @@ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { HttpSetup } from '@kbn/core/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; +import { + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; import { Conversation, Message } from '../../assistant_context/types'; export interface GetConversationByIdParams { @@ -32,10 +37,9 @@ export const getConversationById = async ({ signal, }: GetConversationByIdParams): Promise => { try { - const path = `/api/elastic_assistant/conversations/${id || ''}`; - const response = await http.fetch(path, { + const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { method: 'GET', - version: '2023-10-31', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, }); @@ -67,10 +71,9 @@ export const createConversationApi = async ({ signal, }: PostConversationParams): Promise => { try { - const path = `/api/elastic_assistant/conversations`; - const response = await http.post(path, { + const response = await http.post(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, { body: JSON.stringify(conversation), - version: '2023-10-31', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, }); @@ -106,10 +109,9 @@ export const deleteConversationApi = async ({ signal, }: DeleteConversationParams): Promise => { try { - const path = `/api/elastic_assistant/conversations/${id || ''}`; - const response = await http.fetch(path, { + const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { method: 'DELETE', - version: '2023-10-31', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, }); @@ -152,8 +154,7 @@ export const updateConversationApi = async ({ signal, }: PutConversationMessageParams): Promise => { try { - const path = `/api/elastic_assistant/conversations/${conversationId || ''}`; - const response = await http.fetch(path, { + const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { method: 'PUT', body: JSON.stringify({ id: conversationId, @@ -165,7 +166,7 @@ export const updateConversationApi = async ({ headers: { 'Content-Type': 'application/json', }, - version: '2023-10-31', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts index 1f343c514331f..3fcacc48e883d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts @@ -8,7 +8,11 @@ import { CoreStart } from '@kbn/core/public'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { useQuery } from '@tanstack/react-query'; -// import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; +import { + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; import { Conversation } from '../../assistant_context/types'; export interface FetchConversationsResponse { @@ -18,14 +22,6 @@ export interface FetchConversationsResponse { data: Conversation[]; } -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY = 'elastic_assistant_conversations'; - -const AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; -const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant' as const; -const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations` as const; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = - `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find_user` as const; - export const useFetchCurrentUserConversations = () => { const { http } = useKibana().services; const query = { @@ -33,12 +29,17 @@ export const useFetchCurrentUserConversations = () => { perPage: 100, }; - const querySt = useQuery([ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY, query], () => - http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { - method: 'GET', - version: AI_ASSISTANT_API_CURRENT_VERSION, - query, - }) + const querySt = useQuery( + [ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, query], + () => + http.fetch( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + { + method: 'GET', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + query, + } + ) ); return { ...querySt }; @@ -55,13 +56,12 @@ export const useFetchCurrentUserConversations = () => { * @returns {Promise} */ export const useLastConversation = () => { - const path = `/api/elastic_assistant/conversations/_last_user`; const { http } = useKibana().services; - const querySt = useQuery([path], () => - http.fetch(path, { + const querySt = useQuery([ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST], () => + http.fetch(ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, { method: 'GET', - version: AI_ASSISTANT_API_CURRENT_VERSION, + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, }) ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 3e259993c73c1..606931979173e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -190,7 +190,7 @@ const AssistantComponent: React.FC = ({ ); useEffect(() => { - if (!isLoadingLast && lastConversation) { + if (!isLoadingLast && lastConversation && lastConversation.id) { setCurrentConversation(lastConversation); } }, [isLoadingLast, lastConversation]); diff --git a/x-pack/plugins/elastic_assistant/common/constants.ts b/x-pack/plugins/elastic_assistant/common/constants.ts index 754dfc9211a63..5770ffa6e17e0 100755 --- a/x-pack/plugins/elastic_assistant/common/constants.ts +++ b/x-pack/plugins/elastic_assistant/common/constants.ts @@ -21,18 +21,5 @@ export const EVALUATE = `${BASE_PATH}/evaluate`; export const MAX_CONVERSATIONS_TO_UPDATE_IN_PARALLEL = 50; export const CONVERSATIONS_TABLE_MAX_PAGE_SIZE = 100; -export const ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; -export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_last_user`; - -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_CREATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_create`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_DELETE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_delete`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_UPDATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_update`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find_user`; - // Capabilities export const CAPABILITIES = `${BASE_PATH}/capabilities`; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 930374567533b..0edb31a0f0b49 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -7,9 +7,9 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants'; import { - PostEvaluateBodyInputs, - PostEvaluatePathQueryInputs, -} from '../schemas/evaluate/post_evaluate'; + EvaluateRequestBodyInput, + EvaluateRequestQueryInput, +} from '../schemas/evaluate/post_evaluate_route.gen'; export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -46,8 +46,8 @@ export const getPostEvaluateRequest = ({ body, query, }: { - body: PostEvaluateBodyInputs; - query: PostEvaluatePathQueryInputs; + body: EvaluateRequestBodyInput; + query: EvaluateRequestQueryInput; }) => requestMock.create({ body, diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts index 6d238514b0ba9..8a00ecb029ea3 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts @@ -114,7 +114,7 @@ export function createResourceInstallationHelper( return ( initializedResources.has(key) ? initializedResources.get(key) - : errorResult(`Unrecognized resources key ${key}`) + : errorResult(`Unrecognized spaceId ${key}`) ) as InitializationPromise; }, }; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts deleted file mode 100644 index 17bde15bd01a5..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { getComponentTemplateFromFieldMap } from './component_template_from_field_map'; -import { testFieldMap, expectedTestMapping } from './mapping_from_field_map.test'; - -describe('getComponentTemplateFromFieldMap', () => { - it('correctly creates component template from field map', () => { - expect( - getComponentTemplateFromFieldMap({ name: 'test-mappings', fieldMap: testFieldMap }) - ).toEqual({ - name: 'test-mappings', - _meta: { - managed: true, - }, - template: { - settings: {}, - mappings: { - dynamic: 'strict', - ...expectedTestMapping, - }, - }, - }); - }); - - it('correctly creates component template with settings when includeSettings = true', () => { - expect( - getComponentTemplateFromFieldMap({ - name: 'test-mappings', - fieldMap: testFieldMap, - includeSettings: true, - }) - ).toEqual({ - name: 'test-mappings', - _meta: { - managed: true, - }, - template: { - settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': 1500, - }, - mappings: { - dynamic: 'strict', - ...expectedTestMapping, - }, - }, - }); - }); - - it('correctly creates component template with dynamic setting when defined', () => { - expect( - getComponentTemplateFromFieldMap({ - name: 'test-mappings', - fieldMap: testFieldMap, - includeSettings: true, - dynamic: false, - }) - ).toEqual({ - name: 'test-mappings', - _meta: { - managed: true, - }, - template: { - settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': 1500, - }, - mappings: { - dynamic: false, - ...expectedTestMapping, - }, - }, - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts deleted file mode 100644 index 9d5c651e92cda..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/component_template_from_field_map.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { type FieldMap } from '@kbn/alerts-as-data-utils'; -import { mappingFromFieldMap } from './mapping_from_field_map'; - -export interface GetComponentTemplateFromFieldMapOpts { - name: string; - fieldMap: FieldMap; - includeSettings?: boolean; - dynamic?: 'strict' | false; -} -export const getComponentTemplateFromFieldMap = ({ - name, - fieldMap, - dynamic, - includeSettings, -}: GetComponentTemplateFromFieldMapOpts): ClusterPutComponentTemplateRequest => { - return { - name, - _meta: { - managed: true, - }, - template: { - settings: { - ...(includeSettings - ? { - number_of_shards: 1, - 'index.mapping.total_fields.limit': - Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, - } - : {}), - }, - - mappings: mappingFromFieldMap(fieldMap, dynamic ?? 'strict'), - }, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts deleted file mode 100644 index e58b795863e48..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.test.ts +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; -import { mappingFromFieldMap } from './mapping_from_field_map'; - -export const testFieldMap: FieldMap = { - date_field: { - type: 'date', - array: false, - required: true, - }, - keyword_field: { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - long_field: { - type: 'long', - array: false, - required: false, - }, - multifield_field: { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - multi_fields: [ - { - flat_name: 'multifield_field.text', - name: 'text', - type: 'match_only_text', - }, - ], - }, - geopoint_field: { - type: 'geo_point', - array: false, - required: false, - }, - ip_field: { - type: 'ip', - array: false, - required: false, - }, - array_field: { - type: 'keyword', - array: true, - required: false, - ignore_above: 1024, - }, - nested_array_field: { - type: 'nested', - array: false, - required: false, - }, - 'nested_array_field.field1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'nested_array_field.field2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - scaled_float_field: { - type: 'scaled_float', - array: false, - required: false, - scaling_factor: 1000, - }, - constant_keyword_field: { - type: 'constant_keyword', - array: false, - required: false, - }, - 'parent_field.child1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'parent_field.child2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - unmapped_object: { - type: 'object', - required: false, - enabled: false, - }, - formatted_field: { - type: 'date_range', - required: false, - format: 'epoch_millis||strict_date_optional_time', - }, -}; -export const expectedTestMapping = { - properties: { - array_field: { - ignore_above: 1024, - type: 'keyword', - }, - constant_keyword_field: { - type: 'constant_keyword', - }, - date_field: { - type: 'date', - }, - multifield_field: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - geopoint_field: { - type: 'geo_point', - }, - ip_field: { - type: 'ip', - }, - keyword_field: { - ignore_above: 1024, - type: 'keyword', - }, - long_field: { - type: 'long', - }, - nested_array_field: { - properties: { - field1: { - ignore_above: 1024, - type: 'keyword', - }, - field2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - type: 'nested', - }, - parent_field: { - properties: { - child1: { - ignore_above: 1024, - type: 'keyword', - }, - child2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - scaled_float_field: { - scaling_factor: 1000, - type: 'scaled_float', - }, - unmapped_object: { - enabled: false, - type: 'object', - }, - formatted_field: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - }, -}; - -describe('mappingFromFieldMap', () => { - it('correctly creates mapping from field map', () => { - expect(mappingFromFieldMap(testFieldMap)).toEqual({ - dynamic: 'strict', - ...expectedTestMapping, - }); - expect(mappingFromFieldMap(alertFieldMap)).toEqual({ - dynamic: 'strict', - properties: { - '@timestamp': { - ignore_malformed: false, - type: 'date', - }, - event: { - properties: { - action: { - type: 'keyword', - }, - kind: { - type: 'keyword', - }, - }, - }, - kibana: { - properties: { - alert: { - properties: { - action_group: { - type: 'keyword', - }, - case_ids: { - type: 'keyword', - }, - duration: { - properties: { - us: { - type: 'long', - }, - }, - }, - end: { - type: 'date', - }, - flapping: { - type: 'boolean', - }, - flapping_history: { - type: 'boolean', - }, - maintenance_window_ids: { - type: 'keyword', - }, - instance: { - properties: { - id: { - type: 'keyword', - }, - }, - }, - last_detected: { - type: 'date', - }, - reason: { - type: 'keyword', - }, - rule: { - properties: { - category: { - type: 'keyword', - }, - consumer: { - type: 'keyword', - }, - execution: { - properties: { - uuid: { - type: 'keyword', - }, - }, - }, - name: { - type: 'keyword', - }, - parameters: { - type: 'flattened', - ignore_above: 4096, - }, - producer: { - type: 'keyword', - }, - revision: { - type: 'long', - }, - rule_type_id: { - type: 'keyword', - }, - tags: { - type: 'keyword', - }, - uuid: { - type: 'keyword', - }, - }, - }, - start: { - type: 'date', - }, - status: { - type: 'keyword', - }, - time_range: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - url: { - ignore_above: 2048, - index: false, - type: 'keyword', - }, - uuid: { - type: 'keyword', - }, - workflow_status: { - type: 'keyword', - }, - workflow_tags: { - type: 'keyword', - }, - }, - }, - space_ids: { - type: 'keyword', - }, - version: { - type: 'version', - }, - }, - }, - tags: { - type: 'keyword', - }, - }, - }); - expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ - dynamic: 'strict', - properties: { - kibana: { - properties: { - alert: { - properties: { - risk_score: { type: 'float' }, - rule: { - properties: { - author: { type: 'keyword' }, - created_at: { type: 'date' }, - created_by: { type: 'keyword' }, - description: { type: 'keyword' }, - enabled: { type: 'keyword' }, - from: { type: 'keyword' }, - interval: { type: 'keyword' }, - license: { type: 'keyword' }, - note: { type: 'keyword' }, - references: { type: 'keyword' }, - rule_id: { type: 'keyword' }, - rule_name_override: { type: 'keyword' }, - to: { type: 'keyword' }, - type: { type: 'keyword' }, - updated_at: { type: 'date' }, - updated_by: { type: 'keyword' }, - version: { type: 'keyword' }, - }, - }, - severity: { type: 'keyword' }, - suppression: { - properties: { - docs_count: { type: 'long' }, - end: { type: 'date' }, - terms: { - properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, - }, - start: { type: 'date' }, - }, - }, - system_status: { type: 'keyword' }, - workflow_reason: { type: 'keyword' }, - workflow_user: { type: 'keyword' }, - }, - }, - }, - }, - ecs: { properties: { version: { type: 'keyword' } } }, - }, - }); - }); - - it('uses dynamic setting if specified', () => { - expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ - dynamic: true, - ...expectedTestMapping, - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts deleted file mode 100644 index 1d5121883df69..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/field_maps/mapping_from_field_map.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { set } from '@kbn/safer-lodash-set'; -import type { FieldMap, MultiField } from '@kbn/alerts-as-data-utils'; - -export function mappingFromFieldMap( - fieldMap: FieldMap, - dynamic: 'strict' | boolean = 'strict' -): MappingTypeMapping { - const mappings = { - dynamic, - properties: {}, - }; - - const fields = Object.keys(fieldMap).map((key: string) => { - const field = fieldMap[key]; - return { - name: key, - ...field, - }; - }); - - fields.forEach((field) => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, required, array, multi_fields, ...rest } = field; - const mapped = multi_fields - ? { - ...rest, - // eslint-disable-next-line @typescript-eslint/naming-convention - fields: multi_fields.reduce((acc, multi_field: MultiField) => { - acc[multi_field.name] = { - type: multi_field.type, - }; - return acc; - }, {} as Record), - } - : rest; - - set(mappings.properties, field.name.split('.').join('.properties.'), mapped); - - if (name === '@timestamp') { - set(mappings.properties, `${name}.ignore_malformed`, false); - } - }); - - return mappings; -} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 4100c669a0765..69fcb5b0ba883 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -5,26 +5,14 @@ * 2.0. */ -import type { Metadata } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { - createOrUpdateComponentTemplate, - createOrUpdateIndexTemplate, -} from '@kbn/alerting-plugin/server'; +import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; -// import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; import { AuthenticatedUser } from '@kbn/security-plugin/server'; +import { Subject } from 'rxjs'; import { AssistantResourceNames } from '../types'; -import { - conversationsFieldMap, - getIndexTemplateAndPattern, - totalFieldsLimit, -} from './lib/conversation_configuration_type'; -import { createConcreteWriteIndex } from './lib/create_concrete_write_index'; -import { DataStreamAdapter } from './lib/create_datastream'; -import { AIAssistantDataClient } from '../conversations_data_client'; +import { AIAssistantConversationsDataClient } from '../conversations_data_client'; import { InitializationPromise, ResourceInstallationHelper, @@ -32,49 +20,52 @@ import { errorResult, successResult, } from './create_resource_installation_helper'; -// import { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; -import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; +import { conversationsFieldMap } from './lib/conversation_configuration_type'; + +const TOTAL_FIELDS_LIMIT = 2500; -export const ECS_CONTEXT = `ecs`; function getResourceName(resource: string) { return `.kibana-elastic-ai-assistant-${resource}`; } -// export const getComponentTemplateName = (name: string) => `.alerts-${name}-mappings`; - interface AIAssistantServiceOpts { logger: Logger; kibanaVersion: string; elasticsearchClientPromise: Promise; - dataStreamAdapter: DataStreamAdapter; taskManager: TaskManagerSetupContract; + pluginStop$: Subject; } export interface CreateAIAssistantClientParams { logger: Logger; - namespace: string; + spaceId: string; currentUser: AuthenticatedUser | null; } +export type CreateConversationsDataStream = (params: { + kibanaVersion: string; + spaceId?: string; +}) => DataStreamSpacesAdapter; + export class AIAssistantService { - private dataStreamAdapter: DataStreamAdapter; private initialized: boolean; private isInitializing: boolean = false; - private registeredNamespaces: Set = new Set(); + private conversationsDataStream: DataStreamSpacesAdapter; private resourceInitializationHelper: ResourceInstallationHelper; - private commonInitPromise: Promise; + private initPromise: Promise; constructor(private readonly options: AIAssistantServiceOpts) { this.initialized = false; - this.dataStreamAdapter = options.dataStreamAdapter; + this.conversationsDataStream = this.createConversationsDataStream({ + kibanaVersion: options.kibanaVersion, + }); - this.commonInitPromise = this.initializeResources(); + this.initPromise = this.initializeResources(); - // Create helper for initializing context-specific resources this.resourceInitializationHelper = createResourceInstallationHelper( this.options.logger, - this.commonInitPromise, - this.installAndUpdateNamespaceLevelResources.bind(this) + this.initPromise, + this.installAndUpdateSpaceLevelResources.bind(this) ); } @@ -82,6 +73,51 @@ export class AIAssistantService { return this.initialized; } + private createConversationsDataStream: CreateConversationsDataStream = ({ kibanaVersion }) => { + const conversationsDataStream = new DataStreamSpacesAdapter( + this.resourceNames.aliases.conversations, + { + kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + } + ); + + conversationsDataStream.setComponentTemplate({ + name: this.resourceNames.componentTemplate.conversations, + fieldMap: conversationsFieldMap, + }); + + conversationsDataStream.setIndexTemplate({ + name: this.resourceNames.indexTemplate.conversations, + componentTemplateRefs: [this.resourceNames.componentTemplate.conversations], + }); + + return conversationsDataStream; + }; + + private async initializeResources(): Promise { + this.isInitializing = true; + try { + this.options.logger.debug(`Initializing resources for AIAssistantService`); + const esClient = await this.options.elasticsearchClientPromise; + + await this.conversationsDataStream.install({ + esClient, + logger: this.options.logger, + pluginStop$: this.options.pluginStop$, + }); + + this.initialized = true; + this.isInitializing = false; + return successResult(); + } catch (error) { + this.options.logger.error(`Error initializing AI assistant resources: ${error.message}`); + this.initialized = false; + this.isInitializing = false; + return errorResult(error.message); + } + } + private readonly resourceNames: AssistantResourceNames = { componentTemplate: { conversations: getResourceName('component-template-conversations'), @@ -106,22 +142,22 @@ export class AIAssistantService { public async createAIAssistantDatastreamClient( opts: CreateAIAssistantClientParams - ): Promise { - // Check if context specific installation has succeeded - const { result: initialized, error } = await this.getResourcesInitializationPromise( - opts.namespace + ): Promise { + // Check if resources installation has succeeded + const { result: initialized, error } = await this.getSpaceResourcesInitializationPromise( + opts.spaceId ); - // If initialization failed, retry + // If space evel resources initialization failed, retry if (!initialized && error) { - let initPromise: Promise | undefined; + let initRetryPromise: Promise | undefined; // If !this.initialized, we know that resource initialization failed - // and we need to retry this before retrying the namespace specific resources + // and we need to retry this before retrying the spaceId specific resources if (!this.initialized) { if (!this.isInitializing) { this.options.logger.info(`Retrying common resource initialization`); - initPromise = this.initializeResources(); + initRetryPromise = this.initializeResources(); } else { this.options.logger.info( `Skipped retrying common resource initialization because it is already being retried.` @@ -129,14 +165,14 @@ export class AIAssistantService { } } - this.resourceInitializationHelper.retry(opts.namespace, initPromise); + this.resourceInitializationHelper.retry(opts.spaceId, initRetryPromise); const retryResult = await this.resourceInitializationHelper.getInitializedResources( - opts.namespace ?? DEFAULT_NAMESPACE_STRING + opts.spaceId ?? DEFAULT_NAMESPACE_STRING ); if (!retryResult.result) { - const errorLogPrefix = `There was an error in the framework installing namespace-level resources and creating concrete indices for namespace "${opts.namespace}" - `; + const errorLogPrefix = `There was an error in the framework installing spaceId-level resources and creating concrete indices for spaceId "${opts.spaceId}" - `; // Retry also failed this.options.logger.warn( retryResult.error && error @@ -146,166 +182,47 @@ export class AIAssistantService { return null; } else { this.options.logger.info( - `Resource installation for "${opts.namespace}" succeeded after retry` + `Resource installation for "${opts.spaceId}" succeeded after retry` ); } } - return new AIAssistantDataClient({ + return new AIAssistantConversationsDataClient({ logger: this.options.logger, elasticsearchClientPromise: this.options.elasticsearchClientPromise, - namespace: opts.namespace, + spaceId: opts.spaceId, kibanaVersion: this.options.kibanaVersion, - indexPatternsResorceName: 'kibana-elastic-ai-assistant-conversations', + indexPatternsResorceName: this.resourceNames.aliases.conversations, currentUser: opts.currentUser, }); } - public async getResourcesInitializationPromise( - namespace?: string + public async getSpaceResourcesInitializationPromise( + spaceId: string | undefined = DEFAULT_NAMESPACE_STRING ): Promise { - const registeredOpts = namespace && this.registeredNamespaces.has(namespace) ? namespace : null; - - if (!registeredOpts) { - const errMsg = `Error getting initialized status for namespace ${namespace} - namespace has not been registered.`; - this.options.logger.error(errMsg); - return errorResult(errMsg); - } - - const result = await this.resourceInitializationHelper.getInitializedResources( - namespace ?? DEFAULT_NAMESPACE_STRING - ); - - // If the context is unrecognized and namespace is not the default, we + const result = await this.resourceInitializationHelper.getInitializedResources(spaceId); + // If the spaceId is unrecognized and spaceId is not the default, we // need to kick off resource installation and return the promise if ( result.error && - result.error.includes(`Unrecognized context`) && - namespace !== DEFAULT_NAMESPACE_STRING + result.error.includes(`Unrecognized spaceId`) && + spaceId !== DEFAULT_NAMESPACE_STRING ) { - this.resourceInitializationHelper.add(namespace); - - return this.resourceInitializationHelper.getInitializedResources(namespace ?? 'default'); + this.resourceInitializationHelper.add(spaceId); + return this.resourceInitializationHelper.getInitializedResources(spaceId); } - return result; } - private async initializeResources(): Promise { + private async installAndUpdateSpaceLevelResources( + spaceId: string | undefined = DEFAULT_NAMESPACE_STRING + ) { try { - this.options.logger.debug(`Initializing resources for AIAssistantService`); - const esClient = await this.options.elasticsearchClientPromise; - - await Promise.all([ - /* createOrUpdateComponentTemplate({ - logger: this.options.logger, - esClient, - template: getComponentTemplateFromFieldMap({ - name: `${this.resourceNames.componentTemplate.conversations}-ecs`, - fieldMap: ecsFieldMap, - dynamic: false, - includeSettings: true, - }), - totalFieldsLimit, - }), */ - createOrUpdateComponentTemplate({ - logger: this.options.logger, - esClient, - template: { - name: this.resourceNames.componentTemplate.conversations, - // TODO: add DLM policy - _meta: { - managed: true, - }, - template: { - settings: {}, - mappings: mappingFromFieldMap(conversationsFieldMap, 'strict'), - }, - } as ClusterPutComponentTemplateRequest, - totalFieldsLimit, - }), - ]); - - this.initialized = true; - this.isInitializing = false; - return successResult(); - } catch (error) { - this.options.logger.error(`Error initializing AI assistant resources: ${error.message}`); - this.initialized = false; - this.isInitializing = false; - return errorResult(error.message); - } - } - - private async installAndUpdateNamespaceLevelResources(namespace?: string) { - try { - this.options.logger.debug(`Initializing namespace level resources for AIAssistantService`); - const esClient = await this.options.elasticsearchClientPromise; - - const indexMetadata: Metadata = { - kibana: { - version: this.options.kibanaVersion, - }, - managed: true, - namespace: namespace ?? 'default', - }; - - const indexPatterns = getIndexTemplateAndPattern( - 'kibana-elastic-ai-assistant-conversations', - namespace ?? 'default' - ); - - /* - export const getIndexTemplateAndPattern = ( - context: string, - namespace?: string -): IIndexPatternString => { - const concreteNamespace = namespace ? namespace : DEFAULT_NAMESPACE_STRING; - const pattern = `${context}`; - const patternWithNamespace = `${pattern}-${concreteNamespace}`; - return { - template: `${patternWithNamespace}-index-template`, - pattern: `.internal.${patternWithNamespace}-*`, - basePattern: `.${pattern}-*`, - name: `.internal.${patternWithNamespace}-000001`, - alias: `.${patternWithNamespace}`, - }; -}; -*/ - - await createOrUpdateIndexTemplate({ - logger: this.options.logger, - esClient, - template: { - name: indexPatterns.template, - body: { - data_stream: { hidden: true }, - index_patterns: indexPatterns.pattern, - composed_of: [this.resourceNames.componentTemplate.conversations], - template: { - lifecycle: {}, - settings: { - hidden: true, - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': totalFieldsLimit, - }, - mappings: { - dynamic: false, - _meta: indexMetadata, - }, - }, - _meta: indexMetadata, - }, - }, - }); - - await createConcreteWriteIndex({ - logger: this.options.logger, - esClient, - totalFieldsLimit, - indexPatterns, - dataStreamAdapter: this.dataStreamAdapter, - }); + this.options.logger.debug(`Initializing spaceId level resources for AIAssistantService`); + let indexName = await this.conversationsDataStream.getSpaceIndexName(spaceId); + if (!indexName) { + indexName = await this.conversationsDataStream.installSpace(spaceId); + } } catch (error) { this.options.logger.error( `Error initializing AI assistant namespace level resources: ${error.message}` diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts index c178549c542b4..8131ec8514959 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts @@ -8,24 +8,6 @@ import type { FieldMap } from '@kbn/alerts-as-data-utils'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import { IIndexPatternString } from '../../types'; -const keyword = { - type: 'keyword' as const, - ignore_above: 1024, -}; - -const text = { - type: 'text' as const, -}; - -const date = { - type: 'date' as const, -}; - -const dynamic = { - type: 'object' as const, - dynamic: true, -}; - export const conversationsFieldMap: FieldMap = { '@timestamp': { type: 'date', @@ -164,9 +146,6 @@ export const conversationsFieldMap: FieldMap = { }, } as const; -export const mappingComponentName = '.conversations-mappings'; -export const totalFieldsLimit = 1000; - export const getIndexTemplateAndPattern = ( context: string, namespace?: string @@ -175,10 +154,9 @@ export const getIndexTemplateAndPattern = ( const pattern = `${context}`; const patternWithNamespace = `${pattern}-${concreteNamespace}`; return { - template: `.${patternWithNamespace}-index-template`, - pattern: `.${patternWithNamespace}*`, - basePattern: `.${pattern}-*`, - name: `.${patternWithNamespace}-000001`, - alias: `.${patternWithNamespace}`, + pattern: `${patternWithNamespace}*`, + basePattern: `${pattern}-*`, + name: `${patternWithNamespace}-000001`, + alias: `${patternWithNamespace}`, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts deleted file mode 100644 index 1d6555ea799a1..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.test.ts +++ /dev/null @@ -1,718 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { errors as EsErrors } from '@elastic/elasticsearch'; -import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { createConcreteWriteIndex } from './create_concrete_write_index'; -import { getDataStreamAdapter } from './data_stream_adapter'; - -const randomDelayMultiplier = 0.01; -const logger = loggingSystemMock.createLogger(); -const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - -interface EsError extends Error { - statusCode?: number; - meta?: { - body: { - error: { - type: string; - }; - }; - }; -} - -const GetAliasResponse = { - '.internal.alerts-test.alerts-default-000001': { - aliases: { - alias_1: { - is_hidden: true, - }, - alias_2: { - is_hidden: true, - }, - }, - }, -}; - -const GetDataStreamResponse = { - data_streams: ['any-content-here-means-already-exists'], -} as unknown as IndicesGetDataStreamResponse; - -const SimulateTemplateResponse = { - template: { - aliases: { - alias_name_1: { - is_hidden: true, - }, - alias_name_2: { - is_hidden: true, - }, - }, - mappings: { enabled: false }, - settings: {}, - }, -}; - -const IndexPatterns = { - template: '.alerts-test.alerts-default-index-template', - pattern: '.internal.alerts-test.alerts-default-*', - basePattern: '.alerts-test.alerts-*', - alias: '.alerts-test.alerts-default', - name: '.internal.alerts-test.alerts-default-000001', - validPrefixes: ['.internal.alerts-', '.alerts-'], -}; - -describe('createConcreteWriteIndex', () => { - for (const useDataStream of [false, true]) { - const label = useDataStream ? 'data streams' : 'aliases'; - const dataStreamAdapter = getDataStreamAdapter({ useDataStreamForAlerts: useDataStream }); - - beforeEach(() => { - jest.resetAllMocks(); - jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); - }); - - describe(`using ${label} for alert indices`, () => { - it(`should call esClient to put index template`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({ - name: '.alerts-test.alerts-default', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - } - }); - - it(`should retry on transient ES errors`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); - clusterClient.indices.create - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ - index: '.internal.alerts-test.alerts-default-000001', - shards_acknowledged: true, - acknowledged: true, - }); - clusterClient.indices.createDataStream - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ - acknowledged: true, - }); - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); - } - }); - - it(`should log and throw error if max retries exceeded`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); - clusterClient.indices.create.mockRejectedValue(new EsErrors.ConnectionError('foo')); - clusterClient.indices.createDataStream.mockRejectedValue( - new EsErrors.ConnectionError('foo') - ); - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Error creating data stream .alerts-test.alerts-default - foo` - : `Error creating concrete write index - foo` - ); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(4); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledTimes(4); - } - }); - - it(`should log and throw error if ES throws error`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [] })); - clusterClient.indices.create.mockRejectedValueOnce(new Error('generic error')); - clusterClient.indices.createDataStream.mockRejectedValueOnce(new Error('generic error')); - - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Error creating data stream .alerts-test.alerts-default - generic error` - : `Error creating concrete write index - generic error` - ); - }); - - it(`should log and return if ES throws resource_already_exists_exception error and existing index is already write index`, async () => { - if (useDataStream) return; - - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - const error = new Error(`fail`) as EsError; - error.meta = { - body: { - error: { - type: 'resource_already_exists_exception', - }, - }, - }; - clusterClient.indices.create.mockRejectedValueOnce(error); - clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-000001': { - aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, - }, - })); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - }); - - it(`should retry getting index on transient ES error`, async () => { - if (useDataStream) return; - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - const error = new Error(`fail`) as EsError; - error.statusCode = 404; - error.meta = { - body: { - error: { - type: 'resource_already_exists_exception', - }, - }, - }; - clusterClient.indices.create.mockRejectedValueOnce(error); - clusterClient.indices.get - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-000001': { - aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, - }, - })); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(clusterClient.indices.get).toHaveBeenCalledTimes(3); - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - }); - - it(`should log and throw error if ES throws resource_already_exists_exception error and existing index is not the write index`, async () => { - if (useDataStream) return; - - clusterClient.indices.getAlias.mockImplementation(async () => ({})); - const error = new Error(`fail`) as EsError; - error.meta = { - body: { - error: { - type: 'resource_already_exists_exception', - }, - }, - }; - clusterClient.indices.create.mockRejectedValueOnce(error); - clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-000001': { - aliases: { '.alerts-test.alerts-default': { is_write_index: false } }, - }, - })); - - const ccwiPromise = createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - await expect(() => ccwiPromise).rejects.toThrowErrorMatchingInlineSnapshot( - `"Attempted to create index: .internal.alerts-test.alerts-default-000001 as the write index for alias: .alerts-test.alerts-default, but the index already exists and is not the write index for the alias"` - ); - - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - }); - - it(`should call esClient to put index template if get alias throws 404`, async () => { - const error = new Error(`not found`) as EsError; - error.statusCode = 404; - clusterClient.indices.getAlias.mockRejectedValueOnce(error); - clusterClient.indices.getDataStream.mockRejectedValueOnce(error); - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({ - name: '.alerts-test.alerts-default', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - } - }); - - it(`should log and throw error if get alias throws non-404 error`, async () => { - const error = new Error(`fatal error`) as EsError; - error.statusCode = 500; - clusterClient.indices.getAlias.mockRejectedValueOnce(error); - clusterClient.indices.getDataStream.mockRejectedValueOnce(error); - - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"fatal error"`); - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Error fetching data stream for .alerts-test.alerts-default - fatal error` - : `Error fetching concrete indices for .internal.alerts-test.alerts-default-* pattern - fatal error` - ); - }); - - it(`should update underlying settings and mappings of existing concrete indices if they exist`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (!useDataStream) { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - } - - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 1 : 2); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 1 : 2); - }); - - it(`should skip updating underlying settings and mappings of existing concrete indices if they follow an unexpected naming convention`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => ({ - bad_index_name: { - aliases: { - alias_1: { - is_hidden: true, - }, - }, - }, - })); - - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (!useDataStream) { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - expect(logger.warn).toHaveBeenCalledWith( - `Found unexpected concrete index name "bad_index_name" while expecting index with one of the following prefixes: [.internal.alerts-,.alerts-] Not updating mappings or settings for this index.` - ); - } - - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 1 : 0); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 1 : 0); - }); - - it(`should retry simulateIndexTemplate on transient ES errors`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockImplementation(async () => SimulateTemplateResponse); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( - useDataStream ? 3 : 4 - ); - }); - - it(`should retry getting alias on transient ES errors`, async () => { - clusterClient.indices.getAlias - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - if (useDataStream) { - expect(clusterClient.indices.getDataStream).toHaveBeenCalledTimes(3); - } else { - expect(clusterClient.indices.getAlias).toHaveBeenCalledTimes(3); - } - }); - - it(`should retry settings update on transient ES errors`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putSettings - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 3 : 4); - }); - - it(`should log and throw error on settings update if max retries exceeded`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putSettings.mockRejectedValue(new EsErrors.ConnectionError('foo')); - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(useDataStream ? 4 : 7); - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: foo` - : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: foo` - ); - }); - - it(`should log and throw error on settings update if ES throws error`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putSettings.mockRejectedValue(new Error('generic error')); - - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: generic error` - : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: generic error` - ); - }); - - it(`should retry mappings update on transient ES errors`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putMapping - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 3 : 4); - }); - - it(`should log and throw error on mappings update if max retries exceeded`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putMapping.mockRejectedValue(new EsErrors.ConnectionError('foo')); - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(useDataStream ? 4 : 7); - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Failed to PUT mapping for .alerts-test.alerts-default: foo` - : `Failed to PUT mapping for alias_1: foo` - ); - }); - - it(`should log and throw error on mappings update if ES throws error`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - clusterClient.indices.putMapping.mockRejectedValue(new Error('generic error')); - - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Failed to PUT mapping for .alerts-test.alerts-default: generic error` - : `Failed to PUT mapping for alias_1: generic error` - ); - }); - - it(`should log and return when simulating updated mappings throws error`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Ignored PUT mappings for .alerts-test.alerts-default; error generating simulated mappings: fail` - : `Ignored PUT mappings for alias_1; error generating simulated mappings: fail` - ); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - } - }); - - it(`should log and return when simulating updated mappings returns null`, async () => { - clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); - clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); - clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({ - ...SimulateTemplateResponse, - template: { ...SimulateTemplateResponse.template, mappings: null }, - })); - - await createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }); - - expect(logger.error).toHaveBeenCalledWith( - useDataStream - ? `Ignored PUT mappings for .alerts-test.alerts-default; simulated mappings were empty` - : `Ignored PUT mappings for alias_1; simulated mappings were empty` - ); - - if (useDataStream) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - } - }); - - it(`should throw error when there are concrete indices but none of them are the write index`, async () => { - if (useDataStream) return; - - clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-0001': { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: false, - is_hidden: true, - }, - alias_2: { - is_write_index: false, - is_hidden: true, - }, - }, - }, - })); - clusterClient.indices.simulateIndexTemplate.mockImplementation( - async () => SimulateTemplateResponse - ); - - await expect(() => - createConcreteWriteIndex({ - logger, - esClient: clusterClient, - indexPatterns: IndexPatterns, - totalFieldsLimit: 2500, - dataStreamAdapter, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default"` - ); - }); - }); - } -}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts deleted file mode 100644 index fa87000163f41..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_concrete_write_index.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { Logger, ElasticsearchClient } from '@kbn/core/server'; -import { get } from 'lodash'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; -import { DataStreamAdapter } from './create_datastream'; -import { IIndexPatternString } from '../../types'; - -export interface ConcreteIndexInfo { - index: string; - alias: string; - isWriteIndex: boolean; -} - -interface UpdateIndexMappingsOpts { - logger: Logger; - esClient: ElasticsearchClient; - totalFieldsLimit: number; - validIndexPrefixes?: string[]; - concreteIndices: ConcreteIndexInfo[]; -} - -interface UpdateIndexOpts { - logger: Logger; - esClient: ElasticsearchClient; - totalFieldsLimit: number; - concreteIndexInfo: ConcreteIndexInfo; -} - -const updateTotalFieldLimitSetting = async ({ - logger, - esClient, - totalFieldsLimit, - concreteIndexInfo, -}: UpdateIndexOpts) => { - const { index, alias } = concreteIndexInfo; - try { - await retryTransientEsErrors( - () => - esClient.indices.putSettings({ - index, - body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, - }), - { logger } - ); - } catch (err) { - logger.error( - `Failed to PUT index.mapping.total_fields.limit settings for ${alias}: ${err.message}` - ); - throw err; - } -}; - -// This will update the mappings of backing indices but *not* the settings. This -// is due to the fact settings can be classed as dynamic and static, and static -// updates will fail on an index that isn't closed. New settings *will* be applied as part -// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 -const updateUnderlyingMapping = async ({ - logger, - esClient, - concreteIndexInfo, -}: UpdateIndexOpts) => { - const { index, alias } = concreteIndexInfo; - let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; - try { - simulatedIndexMapping = await retryTransientEsErrors( - () => esClient.indices.simulateIndexTemplate({ name: index }), - { logger } - ); - } catch (err) { - logger.error( - `Ignored PUT mappings for ${alias}; error generating simulated mappings: ${err.message}` - ); - return; - } - - const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); - - if (simulatedMapping == null) { - logger.error(`Ignored PUT mappings for ${alias}; simulated mappings were empty`); - return; - } - - try { - await retryTransientEsErrors( - () => esClient.indices.putMapping({ index, body: simulatedMapping }), - { logger } - ); - } catch (err) { - logger.error(`Failed to PUT mapping for ${alias}: ${err.message}`); - throw err; - } -}; -/** - * Updates the underlying mapping for any existing concrete indices - */ -export const updateIndexMappings = async ({ - logger, - esClient, - totalFieldsLimit, - concreteIndices, - validIndexPrefixes, -}: UpdateIndexMappingsOpts) => { - let validConcreteIndices = []; - if (validIndexPrefixes) { - for (const cIdx of concreteIndices) { - if (!validIndexPrefixes?.some((prefix: string) => cIdx.index.startsWith(prefix))) { - logger.warn( - `Found unexpected concrete index name "${ - cIdx.index - }" while expecting index with one of the following prefixes: [${validIndexPrefixes.join( - ',' - )}] Not updating mappings or settings for this index.` - ); - } else { - validConcreteIndices.push(cIdx); - } - } - } else { - validConcreteIndices = concreteIndices; - } - - logger.debug( - `Updating underlying mappings for ${validConcreteIndices.length} indices / data streams.` - ); - - // Update total field limit setting of found indices - // Other index setting changes are not updated at this time - await Promise.all( - validConcreteIndices.map((index) => - updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) - ) - ); - - // Update mappings of the found indices. - await Promise.all( - validConcreteIndices.map((index) => - updateUnderlyingMapping({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) - ) - ); -}; - -export interface CreateConcreteWriteIndexOpts { - logger: Logger; - esClient: ElasticsearchClient; - totalFieldsLimit: number; - indexPatterns: IIndexPatternString; - dataStreamAdapter: DataStreamAdapter; -} -/** - * Installs index template that uses installed component template - * Prior to installation, simulates the installation to check for possible - * conflicts. Simulate should return an empty mapping if a template - * conflicts with an already installed template. - */ -export const createConcreteWriteIndex = async (opts: CreateConcreteWriteIndexOpts) => { - await opts.dataStreamAdapter.createStream(opts); -}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts deleted file mode 100644 index 6a7ac2ab6df65..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.mock.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DataStreamAdapter } from './create_datastream'; - -export function createDataStreamAdapterMock(): DataStreamAdapter { - return { - getIndexTemplateFields: jest.fn().mockReturnValue({ - index_patterns: ['index-pattern'], - }), - createStream: jest.fn(), - }; -} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts deleted file mode 100644 index f3d87d2d3c5dd..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/create_datastream.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CreateConcreteWriteIndexOpts, updateIndexMappings } from './create_concrete_write_index'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; - -export interface DataStreamAdapter { - getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields; - createStream(opts: CreateConcreteWriteIndexOpts): Promise; -} - -export interface BulkOpProperties { - require_alias: boolean; -} - -export interface IndexTemplateFields { - data_stream?: { hidden: true }; - index_patterns: string[]; - rollover_alias?: string; -} - -export function getDataStreamAdapter(): DataStreamAdapter { - return new DataStreamImplementation(); -} - -// implementation using data streams -class DataStreamImplementation implements DataStreamAdapter { - getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields { - return { - data_stream: { hidden: true }, - index_patterns: [alias], - }; - } - - async createStream(opts: CreateConcreteWriteIndexOpts): Promise { - return createDataStream(opts); - } -} - -async function createDataStream(opts: CreateConcreteWriteIndexOpts): Promise { - const { logger, esClient, indexPatterns, totalFieldsLimit } = opts; - logger.info(`Creating data stream - ${indexPatterns.alias}`); - - // check if data stream exists - let dataStreamExists = false; - try { - const response = await retryTransientEsErrors( - () => esClient.indices.getDataStream({ name: indexPatterns.alias, expand_wildcards: 'all' }), - { logger } - ); - dataStreamExists = response.data_streams.length > 0; - } catch (error) { - if (error?.statusCode !== 404) { - logger.error(`Error fetching data stream for ${indexPatterns.alias} - ${error.message}`); - throw error; - } - } - - // if a data stream exists, update the underlying mapping - if (dataStreamExists) { - await updateIndexMappings({ - logger, - esClient, - totalFieldsLimit, - concreteIndices: [ - { alias: indexPatterns.alias, index: indexPatterns.alias, isWriteIndex: true }, - ], - }); - } else { - try { - await retryTransientEsErrors( - () => - esClient.indices.createDataStream({ - name: indexPatterns.alias, - }), - { logger } - ); - } catch (error) { - if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { - logger.error(`Error creating data stream ${indexPatterns.alias} - ${error.message}`); - throw error; - } - } - } -} diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts deleted file mode 100644 index 37927cf531c93..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/indices.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const elasticAssistantBaseIndexName = 'elastic-assistant'; - -export const allAssistantIndexPattern = '.ds-elastic-assistant*'; - -export const latestAssistantIndexPattern = 'elastic-assistant.elastic-assistant-latest-*'; - -export const getAssistantLatestIndex = (spaceId = 'default') => - `${elasticAssistantBaseIndexName}.elastic-assistant-latest-${spaceId}`; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts deleted file mode 100644 index 7a3839ad3c5bc..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/retry_transient_es_errors.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { errors as EsErrors } from '@elastic/elasticsearch'; - -const MAX_ATTEMPTS = 3; - -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: Error) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -export const retryTransientEsErrors = async ( - esCall: () => Promise, - { - logger, - attempt = 0, - }: { - logger: Logger; - attempt?: number; - } -): Promise => { - try { - return await esCall(); - } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { - const retryCount = attempt + 1; - const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... - - logger.warn( - `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ - e.stack - }` - ); - - // delay with some randomness - await delay(retryDelaySec * 1000 * Math.random()); - return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); - } - - throw e; - } -}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index be783cf0dd61d..34090d74dd6cc 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -36,7 +36,7 @@ export interface ConversationDataWriter { interface ConversationDataWriterOptions { esClient: ElasticsearchClient; index: string; - namespace: string; + spaceId: string; user: { id?: UUID; name?: string }; logger: Logger; } @@ -92,7 +92,7 @@ export class ConversationDataWriter implements ConversationDataWriter { const conversationBody = params.conversationsToCreate?.flatMap((conversation) => [ { create: { _index: this.options.index } }, - transformToCreateScheme(changedAt, this.options.namespace, this.options.user, conversation), + transformToCreateScheme(changedAt, this.options.spaceId, this.options.user, conversation), ]) ?? []; const conversationUpdatedBody = diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts index c6d709eb2214e..35ba89d919a62 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts @@ -25,10 +25,10 @@ export const getLastConversation = async ( }, query: { bool: { - filter: [ - { term: { 'user.id': userId } }, - { term: { excludeFromLastConversationStorage: false } }, - ], + filter: [{ term: { 'user.id': userId } }], + must_not: { + term: { excludeFromLastConversationStorage: false }, + }, }, }, size: 1, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 7c68bf057f3f4..2a23071eb0793 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -32,16 +32,16 @@ export enum OpenAiProviderType { AzureAi = 'Azure OpenAI', } -export interface AIAssistantDataClientParams { +export interface AIAssistantConversationsDataClientParams { elasticsearchClientPromise: Promise; kibanaVersion: string; - namespace: string; + spaceId: string; logger: Logger; indexPatternsResorceName: string; currentUser: AuthenticatedUser | null; } -export class AIAssistantDataClient { +export class AIAssistantConversationsDataClient { /** Kibana space id the conversation are part of */ private readonly spaceId: string; @@ -52,43 +52,40 @@ export class AIAssistantDataClient { private indexTemplateAndPattern: IIndexPatternString; - constructor(private readonly options: AIAssistantDataClientParams) { + constructor(private readonly options: AIAssistantConversationsDataClientParams) { this.indexTemplateAndPattern = getIndexTemplateAndPattern( this.options.indexPatternsResorceName, - this.options.namespace ?? DEFAULT_NAMESPACE_STRING + this.options.spaceId ?? DEFAULT_NAMESPACE_STRING ); this.currentUser = this.options.currentUser; - this.spaceId = this.options.namespace; + this.spaceId = this.options.spaceId; } public async getWriter(): Promise { - const namespace = this.spaceId; - if (this.writerCache.get(namespace)) { - return this.writerCache.get(namespace) as ConversationDataWriter; + const spaceId = this.spaceId; + if (this.writerCache.get(spaceId)) { + return this.writerCache.get(spaceId) as ConversationDataWriter; } const indexPatterns = this.indexTemplateAndPattern; - await this.initializeWriter(namespace, indexPatterns.alias); - return this.writerCache.get(namespace) as ConversationDataWriter; + await this.initializeWriter(spaceId, indexPatterns.alias); + return this.writerCache.get(spaceId) as ConversationDataWriter; } - private async initializeWriter( - namespace: string, - index: string - ): Promise { + private async initializeWriter(spaceId: string, index: string): Promise { const esClient = await this.options.elasticsearchClientPromise; const writer = new ConversationDataWriter({ esClient, - namespace, + spaceId, index, logger: this.options.logger, user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, }); - this.writerCache.set(namespace, writer); + this.writerCache.set(spaceId, writer); return writer; } - public getReader(options: { namespace?: string } = {}) { + public getReader(options: { spaceId?: string } = {}) { const indexPatterns = this.indexTemplateAndPattern.alias; return { diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index 0afa343122273..65ea2054464da 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -5,17 +5,10 @@ * 2.0. */ -import { - PluginInitializerContext, - CoreStart, - Plugin, - Logger, - KibanaRequest, - SavedObjectsClientContract, -} from '@kbn/core/server'; -import { once } from 'lodash'; +import { PluginInitializerContext, CoreStart, Plugin, Logger } from '@kbn/core/server'; import { AssistantFeatures } from '@kbn/elastic-assistant-common'; +import { ReplaySubject, type Subject } from 'rxjs'; import { events } from './lib/telemetry/event_based_telemetry'; import { AssistantTool, @@ -25,26 +18,13 @@ import { ElasticAssistantPluginStart, ElasticAssistantPluginStartDependencies, ElasticAssistantRequestHandlerContext, - GetElser, } from './types'; -import { - deleteKnowledgeBaseRoute, - getKnowledgeBaseStatusRoute, - postActionsConnectorExecuteRoute, - postEvaluateRoute, - postKnowledgeBaseRoute, -} from './routes'; import { AIAssistantService } from './ai_assistant_service'; import { assistantPromptsType, assistantAnonimizationFieldsType } from './saved_object'; -import { - DataStreamAdapter, - getDataStreamAdapter, -} from './ai_assistant_service/lib/create_datastream'; import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; -import { registerConversationsRoutes } from './routes/register_routes'; +import { registerRoutes } from './routes/register_routes'; import { appContextService } from './services/app_context'; -import { getCapabilitiesRoute } from './routes/capabilities/get_capabilities_route'; export class ElasticAssistantPlugin implements @@ -57,12 +37,12 @@ export class ElasticAssistantPlugin { private readonly logger: Logger; private assistantService: AIAssistantService | undefined; - private dataStreamAdapter: DataStreamAdapter; + private pluginStop$: Subject; private readonly kibanaVersion: PluginInitializerContext['env']['packageInfo']['version']; constructor(initializerContext: PluginInitializerContext) { + this.pluginStop$ = new ReplaySubject(1); this.logger = initializerContext.logger.get(); - this.dataStreamAdapter = getDataStreamAdapter(); this.kibanaVersion = initializerContext.env.packageInfo.version; } @@ -76,10 +56,10 @@ export class ElasticAssistantPlugin logger: this.logger.get('service'), taskManager: plugins.taskManager, kibanaVersion: this.kibanaVersion, - dataStreamAdapter: this.dataStreamAdapter, elasticsearchClientPromise: core .getStartServices() .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + pluginStop$: this.pluginStop$, }); const requestContextFactory = new RequestContextFactory({ @@ -101,27 +81,8 @@ export class ElasticAssistantPlugin core.savedObjects.registerType(assistantAnonimizationFieldsType); // this.assistantService registerKBTask + registerRoutes(router, this.logger, plugins); - const getElserId: GetElser = once( - async (request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) => { - return (await plugins.ml.trainedModelsProvider(request, savedObjectsClient).getELSER()) - .model_id; - } - ); - - // Knowledge Base - deleteKnowledgeBaseRoute(router); - getKnowledgeBaseStatusRoute(router, getElserId); - postKnowledgeBaseRoute(router, getElserId); - // Actions Connector Execute (LLM Wrapper) - postActionsConnectorExecuteRoute(router, getElserId); - // Evaluate - postEvaluateRoute(router, getElserId); - // Conversations - registerConversationsRoutes(router, this.logger); - - // Capabilities - getCapabilitiesRoute(router); return { actions: plugins.actions, getRegisteredFeatures: (pluginName: string) => { @@ -159,5 +120,7 @@ export class ElasticAssistantPlugin public stop() { appContextService.stop(); + this.pluginStop$.next(); + this.pluginStop$.complete(); } } diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts index 23723c4c7797c..5e6fbefb77e13 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts @@ -10,11 +10,11 @@ import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/s import { transformError } from '@kbn/securitysolution-es-utils'; import { - CONVERSATIONS_TABLE_MAX_PAGE_SIZE, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; +import { CONVERSATIONS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; @@ -140,7 +140,7 @@ export const bulkActionConversationsRoute = ( request.events.completed$.subscribe(() => abortController.abort()); try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const writer = await dataClient?.getWriter(); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts index c388b63a893b3..77fd6555054d4 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts @@ -10,7 +10,7 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { ConversationCreateProps, @@ -48,7 +48,7 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const createdConversation = await dataClient?.createConversation(request.body); return response.ok({ body: ConversationResponse.parse(createdConversation), diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts index 8fe276f6da966..7a65d5c3e0972 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts @@ -11,7 +11,7 @@ import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; @@ -37,21 +37,16 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => }, }, async (context, request, response): Promise> => { - const siemResponse = buildResponse(response); - /* const validationErrors = validateQueryRuleByIds(request.query); - if (validationErrors.length) { - return siemResponse.error({ statusCode: 400, body: validationErrors }); - }*/ - + const assistantResponse = buildResponse(response); try { const { conversationId } = request.params; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const existingConversation = await dataClient?.getConversation(conversationId); if (existingConversation == null) { - return siemResponse.error({ + return assistantResponse.error({ body: `conversation id: "${conversationId}" not found`, statusCode: 404, }); @@ -61,7 +56,7 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => return response.ok({ body: {} }); } catch (err) { const error = transformError(err); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts index 8a013e9e68cf4..00852a9f8f796 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts @@ -11,7 +11,7 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { FindConversationsRequestQuery, @@ -49,7 +49,7 @@ export const findConversationsRoute = (router: ElasticAssistantPluginRouter, log try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const result = await dataClient?.findConversations({ perPage: query.per_page, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts index bab6e3f9b68e3..1e8703e4340b8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts @@ -11,7 +11,7 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { FindConversationsRequestQuery, @@ -49,7 +49,7 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const currentUser = ctx.elasticAssistant.getCurrentUser(); const additionalFilter = query.filter ? `AND ${query.filter}` : ''; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts index 0a5c06e31702e..fe9f18b6a18e2 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts @@ -9,8 +9,8 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST, -} from '../../../common/constants'; + ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, +} from '@kbn/elastic-assistant-common'; import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; @@ -19,7 +19,7 @@ export const readLastConversationRoute = (router: ElasticAssistantPluginRouter) router.versioned .get({ access: 'public', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_LAST, + path: ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, options: { tags: ['access:elasticAssistant'], }, @@ -34,7 +34,7 @@ export const readLastConversationRoute = (router: ElasticAssistantPluginRouter) try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const conversation = await dataClient?.getLastConversation(); return response.ok({ body: conversation ?? {} }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts index 73782005ce7ff..f44f1cf26b56e 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts @@ -11,7 +11,7 @@ import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; @@ -44,7 +44,7 @@ export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const conversation = await dataClient?.getConversation(conversationId); return response.ok({ body: conversation ?? {} }); } catch (err) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts index fdec45c464725..4f758f6feaeea 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts @@ -11,7 +11,7 @@ import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, -} from '../../../common/constants'; +} from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { @@ -51,7 +51,7 @@ export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantDataClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const existingConversation = await dataClient?.getConversation(conversationId); if (existingConversation == null) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts index 3ae64f1d89f3b..82e1a9eb6e647 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts @@ -10,16 +10,16 @@ import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getPostEvaluateRequest } from '../../__mocks__/request'; import { - PostEvaluateBodyInputs, - PostEvaluatePathQueryInputs, -} from '../../schemas/evaluate/post_evaluate'; + EvaluateRequestQueryInput, + EvaluateRequestBodyInput, +} from '../../schemas/evaluate/post_evaluate_route.gen'; -const defaultBody: PostEvaluateBodyInputs = { +const defaultBody: EvaluateRequestBodyInput = { dataset: undefined, evalPrompt: undefined, }; -const defaultQueryParams: PostEvaluatePathQueryInputs = { +const defaultQueryParams: EvaluateRequestQueryInput = { agents: 'agents', datasetName: undefined, evaluationType: undefined, diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index aa041175b75ee..332d44196c4af 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -11,10 +11,8 @@ import { v4 as uuidv4 } from 'uuid'; import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; -import { buildRouteValidation } from '../../schemas/common'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; import { EVALUATE } from '../../../common/constants'; -import { PostEvaluateBody, PostEvaluatePathQuery } from '../../schemas/evaluate/post_evaluate'; import { performEvaluation } from '../../lib/model_evaluator/evaluation'; import { callAgentExecutor } from '../../lib/langchain/execute_custom_llm_chain'; import { callOpenAIFunctionsExecutor } from '../../lib/langchain/executors/openai_functions_executor'; @@ -30,6 +28,11 @@ import { import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils'; import { RequestBody } from '../../lib/langchain/types'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; +import { + EvaluateRequestBody, + EvaluateRequestQuery, +} from '../../schemas/evaluate/post_evaluate_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; /** * To support additional Agent Executors from the UI, add them to this map @@ -50,8 +53,8 @@ export const postEvaluateRoute = ( { path: EVALUATE, validate: { - body: buildRouteValidation(PostEvaluateBody), - query: buildRouteValidation(PostEvaluatePathQuery), + body: buildRouteValidationWithZod(EvaluateRequestBody), + query: buildRouteValidationWithZod(EvaluateRequestQuery), }, }, async (context, request, response) => { diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts index 550e89667256e..58a6be9ce66b6 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts @@ -12,7 +12,7 @@ import type { Logger } from '@kbn/core/server'; import type { Run } from 'langsmith/schemas'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer } from 'langchain/callbacks'; -import { Dataset } from '../../schemas/evaluate/post_evaluate'; +import { Dataset } from '../../schemas/evaluate/post_evaluate_route.gen'; /** * Returns the LangChain `llmType` for the given connectorId/connectors diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index 8263a2ace49d5..f86236d52d298 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -8,6 +8,8 @@ import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION } from '@kbn/elastic-assistant-common'; import { INVOKE_ASSISTANT_ERROR_EVENT, INVOKE_ASSISTANT_SUCCESS_EVENT, @@ -19,128 +21,140 @@ import { requestHasRequiredAnonymizationParams, } from '../lib/langchain/helpers'; import { buildResponse } from '../lib/build_response'; -import { buildRouteValidation } from '../schemas/common'; -import { - PostActionsConnectorExecuteBody, - PostActionsConnectorExecutePathParams, -} from '../schemas/actions_connector/post_actions_connector_execute'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../types'; import { ESQL_RESOURCE } from './knowledge_base/constants'; import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from './helpers'; +import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { buildRouteValidationWithZod } from './route_validation'; +import { Message } from '../schemas/conversations/common_attributes.gen'; export const postActionsConnectorExecuteRoute = ( router: IRouter, getElser: GetElser ) => { - router.post( - { + router.versioned + .post({ + access: 'internal', path: POST_ACTIONS_CONNECTOR_EXECUTE, - validate: { - body: buildRouteValidation(PostActionsConnectorExecuteBody), - params: buildRouteValidation(PostActionsConnectorExecutePathParams), + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(ExecuteConnectorRequestBody), + params: schema.object({ + connectorId: schema.string(), + }), + }, + }, }, - }, - async (context, request, response) => { - const resp = buildResponse(response); - const assistantContext = await context.elasticAssistant; - const logger: Logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; + async (context, request, response) => { + const resp = buildResponse(response); + const assistantContext = await context.elasticAssistant; + const logger: Logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; - try { - const connectorId = decodeURIComponent(request.params.connectorId); + try { + const connectorId = decodeURIComponent(request.params.connectorId); - // get the actions plugin start contract from the request context: - const actions = (await context.elasticAssistant).actions; + // get the actions plugin start contract from the request context: + const actions = (await context.elasticAssistant).actions; - // if not langchain, call execute action directly and return the response: - if ( - !request.body.isEnabledKnowledgeBase && - !requestHasRequiredAnonymizationParams(request) - ) { - logger.debug('Executing via actions framework directly'); - const result = await executeAction({ actions, request, connectorId }); - telemetry.reportEvent(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, - isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, - }); - return response.ok({ - body: result, - }); - } + // if not langchain, call execute action directly and return the response: + if ( + !request.body.isEnabledKnowledgeBase && + !requestHasRequiredAnonymizationParams(request) + ) { + logger.debug('Executing via actions framework directly'); + const result = await executeAction({ actions, request, connectorId }); + telemetry.reportEvent(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, + isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, + }); + return response.ok({ + body: result, + }); + } - // TODO: Add `traceId` to actions request when calling via langchain - logger.debug( - `Executing via langchain, isEnabledKnowledgeBase: ${request.body.isEnabledKnowledgeBase}, isEnabledRAGAlerts: ${request.body.isEnabledRAGAlerts}` - ); + // TODO: Add `traceId` to actions request when calling via langchain + logger.debug( + `Executing via langchain, isEnabledKnowledgeBase: ${request.body.isEnabledKnowledgeBase}, isEnabledRAGAlerts: ${request.body.isEnabledRAGAlerts}` + ); - // Fetch any tools registered by the request's originating plugin - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const assistantTools = (await context.elasticAssistant).getRegisteredTools(pluginName); + // Fetch any tools registered by the request's originating plugin + const pluginName = getPluginNameFromRequest({ + request, + defaultPluginName: DEFAULT_PLUGIN_NAME, + logger, + }); + const assistantTools = (await context.elasticAssistant).getRegisteredTools(pluginName); - // get a scoped esClient for assistant memory - const esClient = (await context.core).elasticsearch.client.asCurrentUser; + // get a scoped esClient for assistant memory + const esClient = (await context.core).elasticsearch.client.asCurrentUser; - // convert the assistant messages to LangChain messages: - const langChainMessages = getLangChainMessages( - request.body.params.subActionParams.messages - ); + // convert the assistant messages to LangChain messages: + const langChainMessages = getLangChainMessages( + (request.body.params?.subActionParams?.messages ?? []) as Array< + Pick + > + ); - const elserId = await getElser(request, (await context.core).savedObjects.getClient()); + const elserId = await getElser(request, (await context.core).savedObjects.getClient()); - let latestReplacements = { ...request.body.replacements }; - const onNewReplacements = (newReplacements: Record) => { - latestReplacements = { ...latestReplacements, ...newReplacements }; - }; + let latestReplacements = { ...request.body.replacements }; + const onNewReplacements = (newReplacements: Record) => { + latestReplacements = { ...latestReplacements, ...newReplacements }; + }; - const langChainResponseBody = await callAgentExecutor({ - alertsIndexPattern: request.body.alertsIndexPattern, - allow: request.body.allow, - allowReplacement: request.body.allowReplacement, - actions, - isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, - assistantTools, - connectorId, - elserId, - esClient, - kbResource: ESQL_RESOURCE, - langChainMessages, - logger, - onNewReplacements, - request, - replacements: request.body.replacements, - size: request.body.size, - telemetry, - }); + const langChainResponseBody = await callAgentExecutor({ + alertsIndexPattern: request.body.alertsIndexPattern, + allow: request.body.allow, + allowReplacement: request.body.allowReplacement, + actions, + isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase ?? false, + assistantTools, + connectorId, + elserId, + esClient, + kbResource: ESQL_RESOURCE, + langChainMessages, + logger, + onNewReplacements, + request, + replacements: request.body.replacements as Record, + size: request.body.size, + telemetry, + }); - telemetry.reportEvent(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, - isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, - }); - return response.ok({ - body: { - ...langChainResponseBody, - replacements: latestReplacements, - }, - }); - } catch (err) { - logger.error(err); - const error = transformError(err); - telemetry.reportEvent(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { - isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, - isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, - errorMessage: error.message, - }); + telemetry.reportEvent(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, + isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, + }); + return response.ok({ + body: { + ...langChainResponseBody, + replacements: latestReplacements, + }, + }); + } catch (err) { + logger.error(err); + const error = transformError(err); + telemetry.reportEvent(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { + isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, + isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, + errorMessage: error.message, + }); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts new file mode 100644 index 0000000000000..8b4c74b3f295d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_PROMPTS_URL, +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildResponse } from '../utils'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { PromptCreateProps, PromptResponse } from '../../schemas/prompts/crud_prompts_route.gen'; + +export const createPromptRoute = (router: ElasticAssistantPluginRouter): void => { + router.versioned + .post({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL, + + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PromptCreateProps), + }, + }, + }, + async (context, request, response): Promise> => { + const assistantResponse = buildResponse(response); + // const validationErrors = validateCreateRuleProps(request.body); + // if (validationErrors.length) { + // return siemResponse.error({ statusCode: 400, body: validationErrors }); + // } + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + const createdPrompt = await dataClient.createPrompt(request.body); + return response.ok({ + body: PromptResponse.parse(createdPrompt), + }); + } catch (err) { + const error = transformError(err as Error); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts new file mode 100644 index 0000000000000..dd8dbd873dc27 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildResponse } from '../utils'; + +export const deletePromptRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .delete({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + params: schema.object({ + promptId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise => { + const assistantResponse = buildResponse(response); + /* const validationErrors = validateQueryRuleByIds(request.query); + if (validationErrors.length) { + return siemResponse.error({ statusCode: 400, body: validationErrors }); + }*/ + + try { + const { promptId } = request.params; + + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + + const existingPrompt = await dataClient?.getPrompt(promptId); + if (existingPrompt == null) { + return assistantResponse.error({ + body: `prompt id: "${promptId}" not found`, + statusCode: 404, + }); + } + await dataClient?.deletePromptById(promptId); + + return response.ok({ body: {} }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts new file mode 100644 index 0000000000000..00d60a95d6900 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND, +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { + FindPromptsRequestQuery, + FindPromptsResponse, +} from '../../schemas/prompts/find_prompts_route.gen'; + +export const findPromptsRoute = (router: ElasticAssistantPluginRouter, logger: Logger) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + query: buildRouteValidationWithZod(FindPromptsRequestQuery), + }, + }, + }, + async (context, request, response): Promise> => { + const assistantResponse = buildResponse(response); + + try { + const { query } = request; + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + + const result = await dataClient?.findPrompts({ + perPage: query.per_page, + page: query.page, + sortField: query.sort_field, + sortOrder: query.sort_order, + filter: query.filter, + fields: query.fields, + }); + + return response.ok({ body: result }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts new file mode 100644 index 0000000000000..740e990d06583 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { PromptResponse, PromptUpdateProps } from '../../schemas/prompts/crud_prompts_route.gen'; + +export const updatePromptRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .put({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PromptUpdateProps), + params: schema.object({ + promptId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise> => { + const assistantResponse = buildResponse(response); + const { promptId } = request.params; + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + + const existingPrompt = await dataClient?.getPrompt(promptId); + if (existingPrompt == null) { + return assistantResponse.error({ + body: `Prompt id: "${promptId}" not found`, + statusCode: 404, + }); + } + const prompt = await dataClient?.updatePromptItem(existingPrompt, request.body); + if (prompt == null) { + return assistantResponse.error({ + body: `prompt id: "${promptId}" was not updated`, + statusCode: 400, + }); + } + return response.ok({ + body: prompt, + }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index e9a178baf6a35..60ea189b65d10 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -5,9 +5,14 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; +import type { KibanaRequest, Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { ElasticAssistantPluginRouter } from '../types'; +import { once } from 'lodash/fp'; +import { + ElasticAssistantPluginRouter, + ElasticAssistantPluginSetupDependencies, + GetElser, +} from '../types'; import { createConversationRoute } from './conversation/create_route'; import { deleteConversationRoute } from './conversation/delete_route'; import { findConversationsRoute } from './conversation/find_route'; @@ -16,10 +21,21 @@ import { updateConversationRoute } from './conversation/update_route'; import { findUserConversationsRoute } from './conversation/find_user_conversations_route'; import { bulkActionConversationsRoute } from './conversation/bulk_actions_route'; import { readLastConversationRoute } from './conversation/read_last_route'; +import { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; +import { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; +import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; +import { postEvaluateRoute } from './evaluate/post_evaluate'; +import { postActionsConnectorExecuteRoute } from './post_actions_connector_execute'; +import { getCapabilitiesRoute } from './capabilities/get_capabilities_route'; +import { createPromptRoute } from './prompts/create_route'; +import { updatePromptRoute } from './prompts/update_route'; +import { deletePromptRoute } from './prompts/delete_route'; +import { findPromptsRoute } from './prompts/find_route'; -export const registerConversationsRoutes = ( +export const registerRoutes = ( router: ElasticAssistantPluginRouter, - logger: Logger + logger: Logger, + plugins: ElasticAssistantPluginSetupDependencies ) => { // Conversation CRUD createConversationRoute(router); @@ -31,7 +47,31 @@ export const registerConversationsRoutes = ( // Conversations bulk CRUD bulkActionConversationsRoute(router, logger); + // Capabilities + getCapabilitiesRoute(router); + // Conversations search findConversationsRoute(router, logger); findUserConversationsRoute(router); + + // Knowledge Base + deleteKnowledgeBaseRoute(router); + const getElserId: GetElser = once( + async (request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) => { + return (await plugins.ml.trainedModelsProvider(request, savedObjectsClient).getELSER()) + .model_id; + } + ); + getKnowledgeBaseStatusRoute(router, getElserId); + postKnowledgeBaseRoute(router, getElserId); + // Actions Connector Execute (LLM Wrapper) + postActionsConnectorExecuteRoute(router, getElserId); + // Evaluate + postEvaluateRoute(router, getElserId); + + // Prompts + createPromptRoute(router); + findPromptsRoute(router, logger); + updatePromptRoute(router); + deletePromptRoute(router); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index dcf30bc839edb..bf6b9ee6cc5f7 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -16,7 +16,7 @@ import { ElasticAssistantPluginSetupDependencies, ElasticAssistantRequestHandlerContext, } from '../types'; -import { AIAssistantSOClient } from '../saved_object/ai_assistant_so_client'; +import { AIAssistantPromptsSOClient } from '../saved_object/ai_assistant_prompts_so_client'; import { AIAssistantService } from '../ai_assistant_service'; import { appContextService } from '../services/app_context'; @@ -82,20 +82,20 @@ export class RequestContextFactory implements IRequestContextFactory { telemetry: core.analytics, - getAIAssistantSOClient: memoize(() => { + getAIAssistantPromptsSOClient: memoize(() => { const username = startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; - return new AIAssistantSOClient({ + return new AIAssistantPromptsSOClient({ logger: options.logger, user: username, savedObjectsClient: coreContext.savedObjects.client, }); }), - getAIAssistantDataClient: memoize(async () => { + getAIAssistantConversationsDataClient: memoize(async () => { const currentUser = getCurrentUser(); return this.assistantService.createAIAssistantDatastreamClient({ - namespace: getSpaceId(), + spaceId: getSpaceId(), logger: this.logger, currentUser, }); diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts deleted file mode 100644 index c62fcf5495fad..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_default_prompts.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import { assistantPromptsTypeName } from '.'; -import { AIAssistantPrompts } from '../types'; - -export interface SavedObjectsClientArg { - savedObjectsClient: SavedObjectsClientContract; - namespace: string; -} - -const getDefaultAssistantPrompts = ({ namespace }: { namespace: string }): AIAssistantPrompts[] => [ - { id }, -]; - -export const initSavedObjects = async ({ - namespace, - savedObjectsClient, -}: SavedObjectsClientArg & { namespace: string }) => { - const configuration = await getConfigurationSavedObject({ savedObjectsClient }); - if (configuration) { - return configuration; - } - const result = await savedObjectsClient.bulkCreate( - assistantPromptsTypeName, - getDefaultAssistantPrompts({ namespace }), - {} - ); - - const formattedItems = items.map((item) => { - const savedObjectType = getSavedObjectType({ namespaceType: item.namespace_type ?? 'single' }); - const dateNow = new Date().toISOString(); - - return { - attributes: { - comments: [], - created_at: dateNow, - created_by: user, - description: item.description, - entries: item.entries, - name: item.name, - os_types: item.os_types, - tags: item.tags, - tie_breaker_id: tieBreaker ?? uuidv4(), - type: item.type, - updated_by: user, - version: undefined, - }, - type: savedObjectType, - } as { attributes: ExceptionListSoSchema; type: SavedObjectType }; - }); - - const { saved_objects: savedObjects } = - await savedObjectsClient.bulkCreate(formattedItems); - - return result; -}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts similarity index 66% rename from x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts rename to x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts index ce8c5704a25b0..f41b347eb551c 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_so_client.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts @@ -11,61 +11,19 @@ import { type SavedObjectsClientContract, } from '@kbn/core/server'; -import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { + AssistantPromptSoSchema, assistantPromptsTypeName, transformSavedObjectToAssistantPrompt, transformSavedObjectUpdateToAssistantPrompt, + transformSavedObjectsToFoundAssistantPrompt, } from './elastic_assistant_prompts_type'; import { - AssistantPromptSchema, - AssistantPromptSoSchema, - FoundAssistantPromptSchema, - transformSavedObjectsToFoundAssistantPrompt, -} from './assistant_prompts_so_schema'; - -export interface AssistantPromptsCreateOptions { - /** The comments of the endpoint list item */ - content: string; - /** The entries of the endpoint list item */ - promptType: string; - /** The entries of the endpoint list item */ - name: string; - /** The entries of the endpoint list item */ - isDefault?: boolean; - /** The entries of the endpoint list item */ - isNewConversationDefault?: boolean; -} - -export interface AssistantPromptsUpdateOptions { - /** The comments of the endpoint list item */ - content: string; - /** The entries of the endpoint list item */ - promptType: string; - /** The entries of the endpoint list item */ - name: string; - /** The entries of the endpoint list item */ - isDefault?: boolean; - /** The entries of the endpoint list item */ - isNewConversationDefault?: boolean; - id: string; - _version: string; -} - -export interface FindAssistantPromptsOptions { - /** The filter to apply in the search */ - filter?: string; - /** How many per page to return */ - perPage: number; - /** The page number or "undefined" if there is no page number to continue from */ - page: number; - /** The search_after parameter if there is one, otherwise "undefined" can be sent in */ - searchAfter?: SortResults; - /** The sort field string if there is one, otherwise "undefined" can be sent in */ - sortField?: string; - /** The sort order of "asc" or "desc", otherwise "undefined" can be sent in */ - sortOrder?: 'asc' | 'desc'; -} + PromptCreateProps, + PromptResponse, + PromptUpdateProps, +} from '../schemas/prompts/crud_prompts_route.gen'; +import { FindPromptsResponse, SortOrder } from '../schemas/prompts/find_prompts_route.gen'; export interface ConstructorOptions { /** User creating, modifying, deleting, or updating the prompts */ @@ -78,7 +36,7 @@ export interface ConstructorOptions { /** * Class for use for prompts that are used for AI assistant. */ -export class AIAssistantSOClient { +export class AIAssistantPromptsSOClient { /** User creating, modifying, deleting, or updating the prompts */ private readonly user: string; @@ -102,7 +60,7 @@ export class AIAssistantSOClient { * @param options.id the "id" of an exception list * @returns The found exception list or null if none exists */ - public getPrompt = async (id: string): Promise => { + public getPrompt = async (id: string): Promise => { const { savedObjectsClient } = this; if (id != null) { try { @@ -135,7 +93,7 @@ export class AIAssistantSOClient { name, isDefault, isNewConversationDefault, - }: AssistantPromptsCreateOptions): Promise => { + }: PromptCreateProps): Promise => { const { savedObjectsClient, user } = this; const dateNow = new Date().toISOString(); @@ -151,7 +109,7 @@ export class AIAssistantSOClient { is_new_conversation_default: isNewConversationDefault ?? false, prompt_type: promptType, updated_by: user, - version: 1, + updated_at: dateNow, } ); return transformSavedObjectToAssistantPrompt({ savedObject }); @@ -183,40 +141,26 @@ export class AIAssistantSOClient { * @param options.type The type of the endpoint list item (Default is "simple") * @returns The exception list item updated, otherwise null if not updated */ - public updatePromptItem = async ({ - promptType, - content, - name, - isDefault, - isNewConversationDefault, - id, - _version, - }: AssistantPromptsUpdateOptions): Promise => { + public updatePromptItem = async ( + prompt: PromptResponse, + { promptType, content, name, isNewConversationDefault }: PromptUpdateProps + ): Promise => { const { savedObjectsClient, user } = this; - const prompt = await this.getPrompt(id); - if (prompt == null) { - return null; - } else { - const savedObject = await savedObjectsClient.update( - assistantPromptsTypeName, - prompt.id, - { - content, - is_default: isDefault, - is_new_conversation_default: isNewConversationDefault, - prompt_type: promptType, - name, - updated_by: user, - }, - { - version: _version, - } - ); - return transformSavedObjectUpdateToAssistantPrompt({ - prompt, - savedObject, - }); - } + const savedObject = await savedObjectsClient.update( + assistantPromptsTypeName, + prompt.id, + { + content, + is_new_conversation_default: isNewConversationDefault, + prompt_type: promptType, + name, + updated_by: user, + } + ); + return transformSavedObjectUpdateToAssistantPrompt({ + prompt, + savedObject, + }); }; /** @@ -231,35 +175,41 @@ export class AIAssistantSOClient { }; /** - * Finds exception lists given a set of criteria. + * Finds prompts given a set of criteria. * @param options * @param options.filter The filter to apply in the search * @param options.perPage How many per page to return * @param options.page The page number or "undefined" if there is no page number to continue from * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in - * @param options.searchAfter The search_after parameter if there is one, otherwise "undefined" can be sent in * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in - * @returns The found exception lists or null if nothing is found + * @returns The found prompts or null if nothing is found */ public findPrompts = async ({ - filter, perPage, page, - searchAfter, sortField, sortOrder, - }: FindAssistantPromptsOptions): Promise => { + filter, + fields, + }: { + perPage: number; + page: number; + sortField?: string; + sortOrder?: SortOrder; + filter?: string; + fields?: string[]; + }): Promise => { const { savedObjectsClient } = this; const savedObjectsFindResponse = await savedObjectsClient.find({ filter, page, perPage, - searchAfter, sortField, sortOrder, type: assistantPromptsTypeName, + fields, }); return transformSavedObjectsToFoundAssistantPrompt({ savedObjectsFindResponse }); diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts b/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts deleted file mode 100644 index 5039264cff5c3..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/assistant_prompts_so_schema.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -import { versionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { SavedObjectsFindResponse } from '@kbn/core/server'; -import { transformSavedObjectToAssistantPrompt } from './elastic_assistant_prompts_type'; - -export const assistantPromptSoSchema = t.exact( - t.type({ - created_at: t.string, - created_by: t.string, - content: t.string, - is_default: t.boolean, - is_new_conversation_default: t.boolean, - name: t.string, - prompt_type: t.string, - updated_by: t.string, - version: versionOrUndefined, - }) -); - -export type AssistantPromptSoSchema = t.TypeOf; - -export const _version = t.string; -export const _versionOrUndefined = t.union([_version, t.undefined]); - -export const assistantPromptSchema = t.exact( - t.type({ - _version: _versionOrUndefined, - created_at: t.string, - created_by: t.string, - content: t.string, - is_default: t.boolean, - is_new_conversation_default: t.boolean, - name: t.string, - prompt_type: t.string, - updated_by: t.string, - updated_at: t.string, - id: t.string, - version: versionOrUndefined, - }) -); - -export type AssistantPromptSchema = t.TypeOf; - -export const foundAssistantPromptSchema = t.intersection([ - t.exact( - t.type({ - data: t.array(assistantPromptSchema), - page: t.number, - per_page: t.number, - total: t.number, - }) - ), - t.exact(t.partial({})), -]); - -export type FoundAssistantPromptSchema = t.TypeOf; - -export const transformSavedObjectsToFoundAssistantPrompt = ({ - savedObjectsFindResponse, -}: { - savedObjectsFindResponse: SavedObjectsFindResponse; -}): FoundAssistantPromptSchema => { - return { - data: savedObjectsFindResponse.saved_objects.map((savedObject) => - transformSavedObjectToAssistantPrompt({ savedObject }) - ), - page: savedObjectsFindResponse.page, - per_page: savedObjectsFindResponse.per_page, - total: savedObjectsFindResponse.total, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts index bf6be78a1fbb0..5d909bd291215 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts @@ -6,8 +6,14 @@ */ import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; -import type { SavedObject, SavedObjectsType, SavedObjectsUpdateResponse } from '@kbn/core/server'; -import { AssistantPromptSchema, AssistantPromptSoSchema } from './assistant_prompts_so_schema'; +import type { + SavedObject, + SavedObjectsFindResponse, + SavedObjectsType, + SavedObjectsUpdateResponse, +} from '@kbn/core/server'; +import { PromptResponse } from '../schemas/prompts/crud_prompts_route.gen'; +import { FindPromptsResponse } from '../schemas/prompts/find_prompts_route.gen'; export const assistantPromptsTypeName = 'elastic-ai-assistant-prompts'; @@ -16,16 +22,19 @@ export const assistantPromptsTypeMappings: SavedObjectsType['mappings'] = { id: { type: 'keyword', }, - isDefault: { + is_default: { type: 'boolean', }, - isNewConversationDefault: { + is_shared: { + type: 'boolean', + }, + is_new_conversation_default: { type: 'boolean', }, name: { type: 'keyword', }, - promptType: { + prompt_type: { type: 'keyword', }, content: { @@ -54,12 +63,24 @@ export const assistantPromptsType: SavedObjectsType = { mappings: assistantPromptsTypeMappings, }; +export interface AssistantPromptSoSchema { + created_at: string; + created_by: string; + content: string; + is_default?: boolean; + is_shared?: boolean; + is_new_conversation_default?: boolean; + name: string; + prompt_type: string; + updated_at: string; + updated_by: string; +} + export const transformSavedObjectToAssistantPrompt = ({ savedObject, }: { savedObject: SavedObject; -}): AssistantPromptSchema => { - const dateNow = new Date().toISOString(); +}): PromptResponse => { const { version: _version, attributes: { @@ -72,26 +93,23 @@ export const transformSavedObjectToAssistantPrompt = ({ prompt_type, name, updated_by, - version, + updated_at, /* eslint-enable @typescript-eslint/naming-convention */ }, id, - updated_at: updatedAt, } = savedObject; return { - _version, - created_at, - created_by, + createdAt: created_at, + createdBy: created_by, content, - id, name, - prompt_type, - is_default, - is_new_conversation_default, - updated_at: updatedAt ?? dateNow, - updated_by, - version: version ?? 1, + promptType: prompt_type, + isDefault: is_default, + isNewConversationDefault: is_new_conversation_default, + updatedAt: updated_at, + updatedBy: updated_by, + id, }; }; @@ -99,33 +117,48 @@ export const transformSavedObjectUpdateToAssistantPrompt = ({ prompt, savedObject, }: { - prompt: AssistantPromptSchema; + prompt: PromptResponse; savedObject: SavedObjectsUpdateResponse; -}): AssistantPromptSchema => { +}): PromptResponse => { const dateNow = new Date().toISOString(); const { version: _version, - attributes: { name, updated_by: updatedBy, content, prompt_type: promptType, version }, + attributes: { + name, + updated_by: updatedBy, + content, + prompt_type: promptType, + is_new_conversation_default: isNewConversationDefault, + }, id, updated_at: updatedAt, } = savedObject; - // TODO: Change this to do a decode and throw if the saved object is not as expected. - // TODO: Do a throw if after the decode this is not the correct "list_type: list" - // TODO: Update exception list and item types (perhaps separating out) so as to avoid - // defaulting return { - _version, - created_at: prompt.created_at, - created_by: prompt.created_by, + createdAt: prompt.createdAt, + createdBy: prompt.createdBy, content: content ?? prompt.content, - prompt_type: promptType ?? prompt.prompt_type, - version: version ?? prompt.version, + promptType: promptType ?? prompt.promptType, id, - is_default: prompt.is_default, - is_new_conversation_default: prompt.is_new_conversation_default, + isDefault: prompt.isDefault, + isNewConversationDefault: isNewConversationDefault ?? prompt.isNewConversationDefault, name: name ?? prompt.name, - updated_at: updatedAt ?? dateNow, - updated_by: updatedBy ?? prompt.updated_by, + updatedAt: updatedAt ?? dateNow, + updatedBy: updatedBy ?? prompt.updatedBy, + }; +}; + +export const transformSavedObjectsToFoundAssistantPrompt = ({ + savedObjectsFindResponse, +}: { + savedObjectsFindResponse: SavedObjectsFindResponse; +}): FindPromptsResponse => { + return { + data: savedObjectsFindResponse.saved_objects.map((savedObject) => + transformSavedObjectToAssistantPrompt({ savedObject }) + ), + page: savedObjectsFindResponse.page, + perPage: savedObjectsFindResponse.per_page, + total: savedObjectsFindResponse.total, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts deleted file mode 100644 index a03619e6a92f6..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -/** Validates the URL path of a POST request to the `/actions/connector/{connector_id}/_execute` endpoint */ -export const PostActionsConnectorExecutePathParams = t.type({ - connectorId: t.string, -}); - -/** Validates the body of a POST request to the `/actions/connector/{connector_id}/_execute` endpoint */ -export const PostActionsConnectorExecuteBody = t.type({ - params: t.type({ - subActionParams: t.intersection([ - t.type({ - messages: t.array( - t.type({ - // must match ConversationRole from '@kbn/elastic-assistant - role: t.union([t.literal('system'), t.literal('user'), t.literal('assistant')]), - content: t.string, - }) - ), - }), - t.partial({ - model: t.string, - n: t.number, - stop: t.union([t.string, t.array(t.string), t.null]), - temperature: t.number, - }), - ]), - subAction: t.string, - }), - alertsIndexPattern: t.union([t.string, t.undefined]), - allow: t.union([t.array(t.string), t.undefined]), - allowReplacement: t.union([t.array(t.string), t.undefined]), - isEnabledKnowledgeBase: t.boolean, - isEnabledRAGAlerts: t.boolean, - replacements: t.union([t.record(t.string, t.string), t.undefined]), - size: t.union([t.number, t.undefined]), -}); - -export type PostActionsConnectorExecuteBodyInputs = t.TypeOf< - typeof PostActionsConnectorExecuteBody ->; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts new file mode 100644 index 0000000000000..75d03de1de1a5 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +export type ExecuteConnectorRequestParams = z.infer; +export const ExecuteConnectorRequestParams = z.object({ + /** + * The connector's `id` value. + */ + connectorId: z.string(), +}); +export type ExecuteConnectorRequestParamsInput = z.input; + +export type ExecuteConnectorRequestBody = z.infer; +export const ExecuteConnectorRequestBody = z.object({ + params: z + .object({ + subActionParams: z + .object({ + messages: z + .array( + z.object({ + /** + * Message role. + */ + role: z.enum(['system', 'user', 'assistant']).optional(), + content: z.string().optional(), + }) + ) + .optional(), + model: z.string().optional(), + n: z.number().optional(), + stop: z.array(z.string()).optional(), + temperature: z.number().optional(), + }) + .optional(), + subAction: z.string().optional(), + }) + .optional(), + alertsIndexPattern: z.string().optional(), + allow: z.array(z.string()).optional(), + allowReplacement: z.array(z.string()).optional(), + isEnabledKnowledgeBase: z.boolean().optional(), + isEnabledRAGAlerts: z.boolean().optional(), + replacements: z.object({}).catchall(z.unknown()).optional(), + size: z.number().optional(), +}); +export type ExecuteConnectorRequestBodyInput = z.input; + +export type ExecuteConnectorResponse = z.infer; +export const ExecuteConnectorResponse = z.object({ + data: z.string().optional(), + connector_id: z.string().optional(), + replacements: z.object({}).catchall(z.unknown()).optional(), + status: z.string().optional(), + /** + * Trace Data + */ + trace_data: z + .object({ + /** + * Could be any string, not necessarily a UUID + */ + transactionId: z.string().optional(), + /** + * Could be any string, not necessarily a UUID + */ + traceId: z.string().optional(), + }) + .optional(), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml new file mode 100644 index 0000000000000..2e5dd417c4024 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -0,0 +1,118 @@ +openapi: 3.0.0 +info: + title: Execute Connector API endpoint + version: '1' +paths: + /internal/elastic_assistant/actions/connector/{connectorId}/_execute: + post: + operationId: ExecuteConnector + x-codegen-enabled: true + description: Execute Elastic Assistant connector by id + summary: Execute Elastic Assistant connector + tags: + - Connector API + parameters: + - name: connectorId + in: path + required: true + description: The connector's `id` value. + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + params: + type: object + properties: + subActionParams: + type: object + properties: + messages: + type: array + items: + type: object + properties: + role: + type: string + description: Message role. + enum: + - system + - user + - assistant + content: + type: string + model: + type: string + n: + type: number + stop: + type: array + items: + type: string + temperature: + type: number + subAction: + type: string + alertsIndexPattern: + type: string + allow: + type: array + items: + type: string + allowReplacement: + type: array + items: + type: string + isEnabledKnowledgeBase: + type: boolean + isEnabledRAGAlerts: + type: boolean + replacements: + type: object + additionalProperties: true + size: + type: number + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + data: + type: string + connector_id: + type: string + replacements: + type: object + additionalProperties: true + status: + type: string + trace_data: + type: object + description: Trace Data + properties: + transactionId: + type: string + description: Could be any string, not necessarily a UUID + traceId: + type: string + description: Could be any string, not necessarily a UUID + '400': + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts index 0c19e476f0206..70dee1301afbc 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts @@ -6,7 +6,6 @@ */ import { z } from 'zod'; -import { BooleanFromString } from '@kbn/zod-helpers'; /* * NOTICE: Do not edit this file manually. @@ -84,15 +83,6 @@ export const BulkActionBase = z.object({ ids: z.array(z.string()).min(1).optional(), }); -export type PerformBulkActionRequestQuery = z.infer; -export const PerformBulkActionRequestQuery = z.object({ - /** - * Enables dry run mode for the request call. - */ - dry_run: BooleanFromString.optional(), -}); -export type PerformBulkActionRequestQueryInput = z.input; - export type PerformBulkActionRequestBody = z.infer; export const PerformBulkActionRequestBody = z.object({ delete: BulkActionBase.optional(), diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml index 8385f04d0737f..790f4e5e85d5e 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml @@ -11,13 +11,6 @@ paths: description: The bulk action is applied to all conversations that match the filter or to the list of conversations by their IDs. tags: - Bulk API - parameters: - - name: dry_run - in: query - description: Enables dry run mode for the request call. - required: false - schema: - type: boolean requestBody: content: application/json: @@ -41,6 +34,19 @@ paths: application/json: schema: $ref: '#/components/schemas/BulkCrudActionResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string components: schemas: diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts index 1b179c1abb072..48cdb3b73b444 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -76,7 +76,7 @@ export type Replacement = z.infer; export const Replacement = z.object({}).catchall(z.unknown()); /** - * AI assistant sonversation message. + * AI assistant conversation message. */ export type Message = z.infer; export const Message = z.object({ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml index 120b135ece563..ebda501ed1f81 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -56,7 +56,7 @@ components: Message: type: object - description: AI assistant sonversation message. + description: AI assistant conversation message. required: - 'timestamp' - 'content' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts new file mode 100644 index 0000000000000..18f2ee9ce7491 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +import { + ConversationCreateProps, + ConversationResponse, + UUID, + ConversationUpdateProps, +} from './common_attributes.gen'; + +export type CreateConversationRequestBody = z.infer; +export const CreateConversationRequestBody = ConversationCreateProps; +export type CreateConversationRequestBodyInput = z.input; + +export type CreateConversationResponse = z.infer; +export const CreateConversationResponse = ConversationResponse; + +export type DeleteConversationRequestParams = z.infer; +export const DeleteConversationRequestParams = z.object({ + /** + * The conversation's `id` value. + */ + id: UUID, +}); +export type DeleteConversationRequestParamsInput = z.input; + +export type DeleteConversationResponse = z.infer; +export const DeleteConversationResponse = ConversationResponse; + +export type ReadConversationRequestParams = z.infer; +export const ReadConversationRequestParams = z.object({ + /** + * The conversation's `id` value. + */ + id: UUID, +}); +export type ReadConversationRequestParamsInput = z.input; + +export type ReadConversationResponse = z.infer; +export const ReadConversationResponse = ConversationResponse; + +export type UpdateConversationRequestParams = z.infer; +export const UpdateConversationRequestParams = z.object({ + /** + * The conversation's `id` value. + */ + id: UUID, +}); +export type UpdateConversationRequestParamsInput = z.input; + +export type UpdateConversationRequestBody = z.infer; +export const UpdateConversationRequestBody = ConversationUpdateProps; +export type UpdateConversationRequestBodyInput = z.input; + +export type UpdateConversationResponse = z.infer; +export const UpdateConversationResponse = ConversationResponse; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml similarity index 64% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml rename to x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml index 8716d5a0d1e3b..c0b92754102fe 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml @@ -8,6 +8,7 @@ paths: operationId: CreateConversation x-codegen-enabled: true description: Create a conversation + summary: Create a conversation tags: - Conversation API requestBody: @@ -23,12 +24,26 @@ paths: application/json: schema: $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string /api/elastic_assistant/conversations/{id}: get: operationId: ReadConversation x-codegen-enabled: true description: Read a single conversation + summary: Read a single conversation tags: - Conversations API parameters: @@ -45,12 +60,24 @@ paths: application/json: schema: $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string put: operationId: UpdateConversation x-codegen-enabled: true description: Update a single conversation + summary: Update a conversation tags: - Conversation API parameters: @@ -73,12 +100,24 @@ paths: application/json: schema: $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string delete: operationId: DeleteConversation x-codegen-enabled: true description: Deletes a single conversation using the `id` field. + summary: Deletes a single conversation using the `id` field. tags: - Conversation API parameters: @@ -95,3 +134,16 @@ paths: application/json: schema: $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts index 775fc93ae4059..d9c9510655a8f 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts @@ -63,3 +63,42 @@ export const FindConversationsResponse = z.object({ total: z.number().int(), data: z.array(ConversationResponse), }); +export type FindCurrentUserConversationsRequestQuery = z.infer< + typeof FindCurrentUserConversationsRequestQuery +>; +export const FindCurrentUserConversationsRequestQuery = z.object({ + fields: ArrayFromString(z.string()).optional(), + /** + * Search query + */ + filter: z.string().optional(), + /** + * Field to sort by + */ + sort_field: FindConversationsSortField.optional(), + /** + * Sort order + */ + sort_order: SortOrder.optional(), + /** + * Page number + */ + page: z.coerce.number().int().min(1).optional().default(1), + /** + * Conversations per page + */ + per_page: z.coerce.number().int().min(0).optional().default(20), +}); +export type FindCurrentUserConversationsRequestQueryInput = z.input< + typeof FindCurrentUserConversationsRequestQuery +>; + +export type FindCurrentUserConversationsResponse = z.infer< + typeof FindCurrentUserConversationsResponse +>; +export const FindCurrentUserConversationsResponse = z.object({ + page: z.number().int(), + perPage: z.number().int(), + total: z.number().int(), + data: z.array(ConversationResponse), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml index 33fccd5c7a671..b44cebd1d3ec2 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml @@ -7,7 +7,8 @@ paths: get: operationId: FindConversations x-codegen-enabled: true - description: Finds conversationss that match the given query. + description: Finds conversations that match the given query. + summary: Finds conversations that match the given query. tags: - Conversations API parameters: @@ -54,7 +55,7 @@ paths: default: 20 responses: - '200': + 200: description: Successful response content: application/json: @@ -76,6 +77,107 @@ paths: - perPage - total - data + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + + /api/elastic_assistant/conversations/current_user/_find: + get: + operationId: FindCurrentUserConversations + x-codegen-enabled: true + description: Finds current user conversations that match the given query. + summary: Finds current user conversations that match the given query. + tags: + - Conversations API + parameters: + - name: 'fields' + in: query + required: false + schema: + type: array + items: + type: string + - name: 'filter' + in: query + description: Search query + required: false + schema: + type: string + - name: 'sort_field' + in: query + description: Field to sort by + required: false + schema: + $ref: '#/components/schemas/FindConversationsSortField' + - name: 'sort_order' + in: query + description: Sort order + required: false + schema: + $ref: '#/components/schemas/SortOrder' + - name: 'page' + in: query + description: Page number + required: false + schema: + type: integer + minimum: 1 + default: 1 + - name: 'per_page' + in: query + description: Conversations per page + required: false + schema: + type: integer + minimum: 0 + default: 20 + + responses: + 200: + description: Successful response + content: + application/json: + schema: + type: object + properties: + page: + type: integer + perPage: + type: integer + total: + type: integer + data: + type: array + items: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + required: + - page + - perPage + - total + - data + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string components: schemas: diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts deleted file mode 100644 index f520bf9bf93b6..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -/** Validates Output Index starts with `.kibana-elastic-ai-assistant-` */ -const outputIndex = new t.Type( - 'OutputIndexPrefixed', - (input): input is string => - typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-'), - (input, context) => - typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-') - ? t.success(input) - : t.failure( - input, - context, - `Type error: Output Index does not start with '.kibana-elastic-ai-assistant-'` - ), - t.identity -); - -/** Validates the URL path of a POST request to the `/evaluate` endpoint */ -export const PostEvaluatePathQuery = t.type({ - agents: t.string, - datasetName: t.union([t.string, t.undefined]), - evaluationType: t.union([t.string, t.undefined]), - evalModel: t.union([t.string, t.undefined]), - models: t.string, - outputIndex, - projectName: t.union([t.string, t.undefined]), - runName: t.union([t.string, t.undefined]), -}); - -export type PostEvaluatePathQueryInputs = t.TypeOf; - -export type DatasetItem = t.TypeOf; -export const DatasetItem = t.type({ - id: t.union([t.string, t.undefined]), - input: t.string, - reference: t.string, - tags: t.union([t.array(t.string), t.undefined]), - prediction: t.union([t.string, t.undefined]), -}); - -export type Dataset = t.TypeOf; -export const Dataset = t.array(DatasetItem); - -/** Validates the body of a POST request to the `/evaluate` endpoint */ -export const PostEvaluateBody = t.type({ - dataset: t.union([Dataset, t.undefined]), - evalPrompt: t.union([t.string, t.undefined]), -}); - -export type PostEvaluateBodyInputs = t.TypeOf; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts new file mode 100644 index 0000000000000..6d32a8a24f9b7 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +export type DatasetItem = z.infer; +export const DatasetItem = z.object({ + id: z.string(), + input: z.string().optional(), + reference: z.string(), + tags: z.array(z.string()).optional(), + prediction: z.string().optional(), +}); + +export type Dataset = z.infer; +export const Dataset = z.array(DatasetItem); + +export type EvaluateRequestQuery = z.infer; +export const EvaluateRequestQuery = z.object({ + agents: z.string(), + models: z.string(), + outputIndex: z.string(), + datasetName: z.string().optional(), + evaluationType: z.string().optional(), + evalModel: z.string().optional(), + projectName: z.string().optional(), + runName: z.string().optional(), +}); +export type EvaluateRequestQueryInput = z.input; + +export type EvaluateRequestBody = z.infer; +export const EvaluateRequestBody = z.object({ + dataset: Dataset.optional(), + evalPrompt: z.string().optional(), +}); +export type EvaluateRequestBodyInput = z.input; + +export type EvaluateResponse = z.infer; +export const EvaluateResponse = z.object({ + evaluationId: z.number().optional(), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml new file mode 100644 index 0000000000000..4d7e189216094 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml @@ -0,0 +1,108 @@ +openapi: 3.0.0 +info: + title: Evaluate API endpoint + version: '2023-10-31' +paths: + /internal/elastic_assistant/evaluate: + get: + operationId: Evaluate + x-codegen-enabled: true + description: Get Elastic Assistant capabilities for the requesting plugin + summary: Get Elastic Assistant capabilities + tags: + - Evaluate API + parameters: + - in: query + name: agents + required: true + schema: + type: string + - in: query + name: models + required: true + schema: + type: string + - in: query + name: outputIndex + required: true + schema: + type: string + - in: query + name: datasetName + schema: + type: string + - in: query + name: evaluationType + schema: + type: string + - in: query + name: evalModel + schema: + type: string + - in: query + name: projectName + schema: + type: string + - in: query + name: runName + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + dataset: + $ref: '#/components/schemas/Dataset' + evalPrompt: + type: string + + responses: + 200: + description: Successful response + content: + application/json: + schema: + type: object + properties: + evaluationId: + type: number + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + DatasetItem: + type: object + required: + - id + - reference + properties: + id: + type: string + input: + type: string + reference: + type: string + tags: + type: array + items: + type: string + prediction: + type: string + Dataset: + type: array + items: + $ref: '#/components/schemas/DatasetItem' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml deleted file mode 100644 index 8716d5a0d1e3b..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yml +++ /dev/null @@ -1,97 +0,0 @@ -openapi: 3.0.0 -info: - title: Create Conversation API endpoint - version: '2023-10-31' -paths: - /api/elastic_assistant/conversations: - post: - operationId: CreateConversation - x-codegen-enabled: true - description: Create a conversation - tags: - - Conversation API - requestBody: - required: true - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - get: - operationId: ReadConversation - x-codegen-enabled: true - description: Read a single conversation - tags: - - Conversations API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - put: - operationId: UpdateConversation - x-codegen-enabled: true - description: Update a single conversation - tags: - - Conversation API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - requestBody: - required: true - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - delete: - operationId: DeleteConversation - x-codegen-enabled: true - description: Deletes a single conversation using the `id` field. - tags: - - Conversation API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts new file mode 100644 index 0000000000000..b3dd9c2211cc1 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +/** + * AI assistant create KB settings. + */ +export type KnowledgeBaseCreateProps = z.infer; +export const KnowledgeBaseCreateProps = z.object({ + /** + * Prompt content. + */ + content: z.string(), + /** + * Prompt type. + */ + promptType: z.string(), + /** + * Is default prompt. + */ + isDefault: z.boolean().optional(), + /** + * Is shared prompt. + */ + isShared: z.boolean().optional(), + /** + * Is default prompt. + */ + isNewConversationDefault: z.boolean().optional(), +}); + +/** + * AI assistant KnowledgeBase. + */ +export type KnowledgeBaseResponse = z.infer; +export const KnowledgeBaseResponse = z.object({ + /** + * Prompt content. + */ + content: z.string(), + /** + * Prompt type. + */ + promptType: z.string().optional(), + /** + * Is default prompt. + */ + isDefault: z.boolean().optional(), + /** + * Is shared prompt. + */ + isShared: z.boolean().optional(), + /** + * Is default prompt. + */ + isNewConversationDefault: z.boolean().optional(), +}); + +export type CreateKnowledgeBaseRequestParams = z.infer; +export const CreateKnowledgeBaseRequestParams = z.object({ + /** + * The KnowledgeBase `resource` value. + */ + resource: z.string(), +}); +export type CreateKnowledgeBaseRequestParamsInput = z.input< + typeof CreateKnowledgeBaseRequestParams +>; + +export type CreateKnowledgeBaseRequestBody = z.infer; +export const CreateKnowledgeBaseRequestBody = KnowledgeBaseCreateProps; +export type CreateKnowledgeBaseRequestBodyInput = z.input; + +export type CreateKnowledgeBaseResponse = z.infer; +export const CreateKnowledgeBaseResponse = KnowledgeBaseResponse; + +export type DeleteKnowledgeBaseRequestParams = z.infer; +export const DeleteKnowledgeBaseRequestParams = z.object({ + /** + * The KnowledgeBase `resource` value. + */ + resource: z.string(), +}); +export type DeleteKnowledgeBaseRequestParamsInput = z.input< + typeof DeleteKnowledgeBaseRequestParams +>; + +export type DeleteKnowledgeBaseResponse = z.infer; +export const DeleteKnowledgeBaseResponse = KnowledgeBaseResponse; + +export type ReadKnowledgeBaseRequestParams = z.infer; +export const ReadKnowledgeBaseRequestParams = z.object({ + /** + * The KnowledgeBase `resource` value. + */ + resource: z.string(), +}); +export type ReadKnowledgeBaseRequestParamsInput = z.input; + +export type ReadKnowledgeBaseResponse = z.infer; +export const ReadKnowledgeBaseResponse = KnowledgeBaseResponse; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml new file mode 100644 index 0000000000000..b3ab6e66af62d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml @@ -0,0 +1,163 @@ +openapi: 3.0.0 +info: + title: Create Conversation API endpoint + version: '2023-10-31' +paths: + /internal/elastic_assistant/knowledge_base/{resource}: + post: + operationId: CreateKnowledgeBase + x-codegen-enabled: true + summary: Create a KnowledgeBase + description: Create a KnowledgeBase + tags: + - KnowledgeBase API + parameters: + - name: resource + in: path + required: true + description: The KnowledgeBase `resource` value. + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/KnowledgeBaseCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/KnowledgeBaseResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + get: + operationId: ReadKnowledgeBase + x-codegen-enabled: true + description: Read a single KB + summary: Read a KnowledgeBase + tags: + - KnowledgeBase API + parameters: + - name: resource + in: path + required: true + description: The KnowledgeBase `resource` value. + schema: + type: string + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/KnowledgeBaseResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + delete: + operationId: DeleteKnowledgeBase + x-codegen-enabled: true + description: Deletes KnowledgeBase using the `resource` field. + summary: Deletes a KnowledgeBase + tags: + - KnowledgeBase API + parameters: + - name: resource + in: path + required: true + description: The KnowledgeBase `resource` value. + schema: + type: string + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/KnowledgeBaseResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + KnowledgeBaseCreateProps: + type: object + description: AI assistant create KB settings. + required: + - 'content' + - 'promptType' + properties: + content: + type: string + description: Prompt content. + promptType: + type: string + description: Prompt type. + isDefault: + description: Is default prompt. + type: boolean + isShared: + description: Is shared prompt. + type: boolean + isNewConversationDefault: + description: Is default prompt. + type: boolean + + KnowledgeBaseResponse: + type: object + description: AI assistant KnowledgeBase. + required: + - 'timestamp' + - 'content' + - 'role' + properties: + content: + type: string + description: Prompt content. + promptType: + type: string + description: Prompt type. + isDefault: + description: Is default prompt. + type: boolean + isShared: + description: Is shared prompt. + type: boolean + isNewConversationDefault: + description: Is default prompt. + type: boolean diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml deleted file mode 100644 index 8716d5a0d1e3b..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yml +++ /dev/null @@ -1,97 +0,0 @@ -openapi: 3.0.0 -info: - title: Create Conversation API endpoint - version: '2023-10-31' -paths: - /api/elastic_assistant/conversations: - post: - operationId: CreateConversation - x-codegen-enabled: true - description: Create a conversation - tags: - - Conversation API - requestBody: - required: true - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationCreateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - get: - operationId: ReadConversation - x-codegen-enabled: true - description: Read a single conversation - tags: - - Conversations API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - put: - operationId: UpdateConversation - x-codegen-enabled: true - description: Update a single conversation - tags: - - Conversation API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - requestBody: - required: true - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationUpdateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' - - /api/elastic_assistant/conversations/{id}: - delete: - operationId: DeleteConversation - x-codegen-enabled: true - description: Deletes a single conversation using the `id` field. - tags: - - Conversation API - parameters: - - name: id - in: path - required: true - description: The conversation's `id` value. - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/UUID' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts new file mode 100644 index 0000000000000..c8c2e8d339945 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +/** + * AI assistant create prompt settings. + */ +export type PromptCreateProps = z.infer; +export const PromptCreateProps = z.object({ + /** + * Prompt content. + */ + content: z.string(), + /** + * Prompt name. + */ + name: z.string(), + /** + * Prompt type. + */ + promptType: z.string(), + /** + * Is default prompt. + */ + isDefault: z.boolean().optional(), + /** + * Is shared prompt. + */ + isShared: z.boolean().optional(), + /** + * Is default prompt. + */ + isNewConversationDefault: z.boolean().optional(), +}); + +/** + * AI assistant update prompt settings. + */ +export type PromptUpdateProps = z.infer; +export const PromptUpdateProps = z.object({ + /** + * Prompt content. + */ + content: z.string().optional(), + /** + * Prompt name. + */ + name: z.string().optional(), + /** + * Prompt type. + */ + promptType: z.string().optional(), + /** + * Is shared prompt. + */ + isShared: z.boolean().optional(), + /** + * Is default prompt. + */ + isNewConversationDefault: z.boolean().optional(), +}); + +/** + * AI assistant prompt. + */ +export type PromptResponse = z.infer; +export const PromptResponse = z.object({ + id: z.string(), + /** + * Prompt content. + */ + content: z.string(), + /** + * Prompt name. + */ + name: z.string().optional(), + /** + * Prompt type. + */ + promptType: z.string().optional(), + /** + * Is default prompt. + */ + isDefault: z.boolean().optional(), + /** + * Is shared prompt. + */ + isShared: z.boolean().optional(), + /** + * Is default prompt. + */ + isNewConversationDefault: z.boolean().optional(), + /** + * The last time prompt was updated. + */ + updatedAt: z.string().optional(), + /** + * The last time prompt was updated. + */ + createdAt: z.string().optional(), + /** + * User who was updated prompt. + */ + updatedBy: z.string().optional(), + /** + * User who was created prompt. + */ + createdBy: z.string().optional(), +}); + +export type CreatePromptRequestBody = z.infer; +export const CreatePromptRequestBody = PromptCreateProps; +export type CreatePromptRequestBodyInput = z.input; + +export type CreatePromptResponse = z.infer; +export const CreatePromptResponse = PromptResponse; + +export type DeletePromptRequestParams = z.infer; +export const DeletePromptRequestParams = z.object({ + /** + * The prompt's `id` value. + */ + id: z.string(), +}); +export type DeletePromptRequestParamsInput = z.input; + +export type DeletePromptResponse = z.infer; +export const DeletePromptResponse = PromptResponse; + +export type ReadPromptRequestParams = z.infer; +export const ReadPromptRequestParams = z.object({ + /** + * The prompt's `id` value. + */ + id: z.string(), +}); +export type ReadPromptRequestParamsInput = z.input; + +export type ReadPromptResponse = z.infer; +export const ReadPromptResponse = PromptResponse; + +export type UpdatePromptRequestParams = z.infer; +export const UpdatePromptRequestParams = z.object({ + /** + * The prompt's `id` value. + */ + id: z.string(), +}); +export type UpdatePromptRequestParamsInput = z.input; + +export type UpdatePromptRequestBody = z.infer; +export const UpdatePromptRequestBody = PromptUpdateProps; +export type UpdatePromptRequestBodyInput = z.input; + +export type UpdatePromptResponse = z.infer; +export const UpdatePromptResponse = PromptResponse; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml new file mode 100644 index 0000000000000..39736d61a3786 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml @@ -0,0 +1,240 @@ +openapi: 3.0.0 +info: + title: Create Prompt API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/prompts: + post: + operationId: CreatePrompt + x-codegen-enabled: true + description: Create a prompt + summary: Create a prompt + tags: + - Prompt API + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PromptCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/PromptResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + + /api/elastic_assistant/prompts/{id}: + get: + operationId: ReadPrompt + x-codegen-enabled: true + description: Read a single prompt + summary: Read a single prompt + tags: + - Prompts API + parameters: + - name: id + in: path + required: true + description: The prompt's `id` value. + schema: + type: string + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/PromptResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + put: + operationId: UpdatePrompt + x-codegen-enabled: true + description: Update a single prompt + summary: Update a single prompt + tags: + - Prompt API + parameters: + - name: id + in: path + required: true + description: The prompt's `id` value. + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PromptUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/PromptResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + delete: + operationId: DeletePrompt + x-codegen-enabled: true + description: Deletes a single prompt using the `id` field. + summary: Deletes a single prompt + tags: + - Prompt API + parameters: + - name: id + in: path + required: true + description: The prompt's `id` value. + schema: + type: string + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/PromptResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + PromptCreateProps: + type: object + description: AI assistant create prompt settings. + required: + - 'content' + - 'name' + - 'promptType' + properties: + content: + type: string + description: Prompt content. + name: + type: string + description: Prompt name. + promptType: + type: string + description: Prompt type. + isDefault: + description: Is default prompt. + type: boolean + isShared: + description: Is shared prompt. + type: boolean + isNewConversationDefault: + description: Is default prompt. + type: boolean + + PromptUpdateProps: + type: object + description: AI assistant update prompt settings. + properties: + content: + type: string + description: Prompt content. + name: + type: string + description: Prompt name. + promptType: + type: string + description: Prompt type. + isShared: + description: Is shared prompt. + type: boolean + isNewConversationDefault: + description: Is default prompt. + type: boolean + + PromptResponse: + type: object + description: AI assistant prompt. + required: + - 'timestamp' + - 'content' + - 'role' + - 'id' + properties: + id: + type: string + content: + type: string + description: Prompt content. + name: + type: string + description: Prompt name. + promptType: + type: string + description: Prompt type. + isDefault: + description: Is default prompt. + type: boolean + isShared: + description: Is shared prompt. + type: boolean + isNewConversationDefault: + description: Is default prompt. + type: boolean + updatedAt: + description: The last time prompt was updated. + type: string + createdAt: + description: The last time prompt was updated. + type: string + updatedBy: + description: User who was updated prompt. + type: string + createdBy: + description: User who was created prompt. + type: string diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts new file mode 100644 index 0000000000000..4c758384c8a2d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { ArrayFromString } from '@kbn/zod-helpers'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +import { PromptResponse } from './crud_prompts_route.gen'; + +export type FindPromptsSortField = z.infer; +export const FindPromptsSortField = z.enum(['created_at', 'is_default', 'title', 'updated_at']); +export type FindPromptsSortFieldEnum = typeof FindPromptsSortField.enum; +export const FindPromptsSortFieldEnum = FindPromptsSortField.enum; + +export type SortOrder = z.infer; +export const SortOrder = z.enum(['asc', 'desc']); +export type SortOrderEnum = typeof SortOrder.enum; +export const SortOrderEnum = SortOrder.enum; + +export type FindPromptsRequestQuery = z.infer; +export const FindPromptsRequestQuery = z.object({ + fields: ArrayFromString(z.string()).optional(), + /** + * Search query + */ + filter: z.string().optional(), + /** + * Field to sort by + */ + sort_field: FindPromptsSortField.optional(), + /** + * Sort order + */ + sort_order: SortOrder.optional(), + /** + * Page number + */ + page: z.coerce.number().int().min(1).optional().default(1), + /** + * Prompts per page + */ + per_page: z.coerce.number().int().min(0).optional().default(20), +}); +export type FindPromptsRequestQueryInput = z.input; + +export type FindPromptsResponse = z.infer; +export const FindPromptsResponse = z.object({ + page: z.number().int(), + perPage: z.number().int(), + total: z.number().int(), + data: z.array(PromptResponse), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml new file mode 100644 index 0000000000000..fb197c8b42476 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml @@ -0,0 +1,108 @@ +openapi: 3.0.0 +info: + title: Find Prompts API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/prompts/_find: + get: + operationId: FindPrompts + x-codegen-enabled: true + description: Finds prompts that match the given query. + summary: Finds prompts that match the given query. + tags: + - Prompts API + parameters: + - name: 'fields' + in: query + required: false + schema: + type: array + items: + type: string + - name: 'filter' + in: query + description: Search query + required: false + schema: + type: string + - name: 'sort_field' + in: query + description: Field to sort by + required: false + schema: + $ref: '#/components/schemas/FindPromptsSortField' + - name: 'sort_order' + in: query + description: Sort order + required: false + schema: + $ref: '#/components/schemas/SortOrder' + - name: 'page' + in: query + description: Page number + required: false + schema: + type: integer + minimum: 1 + default: 1 + - name: 'per_page' + in: query + description: Prompts per page + required: false + schema: + type: integer + minimum: 0 + default: 20 + + responses: + 200: + description: Successful response + content: + application/json: + schema: + type: object + properties: + page: + type: integer + perPage: + type: integer + total: + type: integer + data: + type: array + items: + $ref: './crud_prompts_route.schema.yaml#/components/schemas/PromptResponse' + required: + - page + - perPage + - total + - data + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + FindPromptsSortField: + type: string + enum: + - 'created_at' + - 'is_default' + - 'title' + - 'updated_at' + + SortOrder: + type: string + enum: + - 'asc' + - 'desc' diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index 79ba37cda689c..359441b02ea63 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -26,9 +26,9 @@ import { AuthenticatedUser, SecurityPluginStart } from '@kbn/security-plugin/ser import { Tool } from 'langchain/dist/tools/base'; import { RetrievalQAChain } from 'langchain/chains'; import { ElasticsearchClient } from '@kbn/core/server'; -import { AIAssistantDataClient } from './conversations_data_client'; -import { AIAssistantSOClient } from './saved_object/ai_assistant_so_client'; import { AssistantFeatures } from '@kbn/elastic-assistant-common'; +import { AIAssistantConversationsDataClient } from './conversations_data_client'; +import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; import { RequestBody } from './lib/langchain/types'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; @@ -98,8 +98,8 @@ export interface ElasticAssistantApiRequestHandlerContext { getServerBasePath: () => string; getSpaceId: () => string; getCurrentUser: () => AuthenticatedUser | null; - getAIAssistantDataClient: () => Promise; - getAIAssistantSOClient: () => AIAssistantSOClient; + getAIAssistantConversationsDataClient: () => Promise; + getAIAssistantPromptsSOClient: () => AIAssistantPromptsSOClient; telemetry: AnalyticsServiceSetup; } /** @@ -151,7 +151,6 @@ export interface AssistantResourceNames { } export interface IIndexPatternString { - template: string; pattern: string; alias: string; name: string; diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index dfca7893b2036..b1a401d8cfcd1 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -35,6 +35,7 @@ "@kbn/core-analytics-server", "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", + "@kbn/data-stream-adapter", ], "exclude": [ "target/**/*", From dabe489c3be58312325eb9b9fde60f8248b960f5 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 11:22:11 -0800 Subject: [PATCH 008/141] migrated existing apis to versioned api --- .../kbn-elastic-assistant-common/constants.ts | 9 +- .../api/{ => conversations}/conversations.ts | 40 +- .../assistant/api/conversations/index.tsx} | 9 +- .../use_bulk_actions_conversations.ts | 2 +- .../use_fetch_current_user_conversations.ts | 2 +- .../impl/assistant/api/index.tsx | 8 + .../assistant/assistant_overlay/index.tsx | 2 +- .../impl/assistant/index.tsx | 2 +- .../use_settings_updater.tsx | 2 +- .../packages/kbn-elastic-assistant/index.ts | 2 +- .../sub_action_connector.ts | 1 + .../elastic_assistant/common/constants.ts | 3 + .../conversation_configuration_type.ts | 2 +- .../server/ai_assistant_service/index.ts | 2 +- .../lib/get_configuration_index.test.ts | 59 --- .../lib/get_configuration_index.ts | 44 --- .../server/conversations_data_client/index.ts | 2 +- .../bulk_actions_route.ts | 191 +++++++++ .../routes/anonimization_fields/find_route.ts | 75 ++++ .../server/routes/evaluate/post_evaluate.ts | 372 +++++++++--------- .../knowledge_base/delete_knowledge_base.ts | 110 +++--- .../routes/knowledge_base/get_kb_resource.ts | 14 +- .../get_knowledge_base_status.ts | 116 +++--- .../knowledge_base/post_knowledge_base.ts | 143 ++++--- .../routes/post_actions_connector_execute.ts | 4 +- .../server/routes/request_context_factory.ts | 11 + ...ssistant_anonimization_fields_so_client.ts | 244 ++++++++++++ ...tic_assistant_anonimization_fields_type.ts | 110 +++++- ...ulk_crud_anonimization_fields_route.gen.ts | 115 ++++++ ...rud_anonimization_fields_route.schema.yaml | 230 +++++++++++ .../find_prompts_route.gen.ts | 69 ++++ .../find_prompts_route.schema.yaml | 108 +++++ .../server/schemas/common.ts | 38 -- .../knowledge_base/crud_kb_route.gen.ts | 57 +-- .../knowledge_base/crud_kb_route.schema.yaml | 62 +-- .../knowledge_base/delete_knowledge_base.ts | 13 - .../get_knowledge_base_status.ts | 13 - .../plugins/elastic_assistant/server/types.ts | 2 + 38 files changed, 1622 insertions(+), 666 deletions(-) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/api/{ => conversations}/conversations.ts (85%) rename x-pack/{plugins/elastic_assistant/server/schemas/knowledge_base/post_knowledge_base.ts => packages/kbn-elastic-assistant/impl/assistant/api/conversations/index.tsx} (53%) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/api/{ => conversations}/use_bulk_actions_conversations.ts (95%) rename x-pack/packages/kbn-elastic-assistant/impl/assistant/api/{ => conversations}/use_fetch_current_user_conversations.ts (96%) rename x-pack/plugins/elastic_assistant/server/ai_assistant_service/{lib => }/conversation_configuration_type.ts (98%) delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts create mode 100644 x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/common.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/delete_knowledge_base.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/get_knowledge_base_status.ts diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index 7b75a303ab0e9..959852fb9c4a7 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -6,6 +6,8 @@ */ export const ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION = '2023-10-31'; +export const ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION = '1'; + export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; @@ -13,9 +15,6 @@ export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL = `${ELASTIC_AI export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_last`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_CREATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_create`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_DELETE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_delete`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_UPDATE = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_update`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_find`; @@ -23,3 +22,7 @@ export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `$ export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/prompts`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{promptId}`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_find`; + +export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL = `${ELASTIC_AI_ASSISTANT_URL}/anonimization_fields`; +export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL}/_bulk_action`; +export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND = `${ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL}/_find`; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts similarity index 85% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts index c8c67024aa671..60f69dcd8b783 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts @@ -10,10 +10,9 @@ import { HttpSetup } from '@kbn/core/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; -import { Conversation, Message } from '../../assistant_context/types'; +import { Conversation, Message } from '../../../assistant_context/types'; export interface GetConversationByIdParams { http: HttpSetup; @@ -37,7 +36,7 @@ export const getConversationById = async ({ signal, }: GetConversationByIdParams): Promise => { try { - const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { + const response = await http.fetch(`${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${id}`, { method: 'GET', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, @@ -109,7 +108,7 @@ export const deleteConversationApi = async ({ signal, }: DeleteConversationParams): Promise => { try { - const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { + const response = await http.fetch(`${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${id}`, { method: 'DELETE', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, signal, @@ -154,21 +153,24 @@ export const updateConversationApi = async ({ signal, }: PutConversationMessageParams): Promise => { try { - const response = await http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, { - method: 'PUT', - body: JSON.stringify({ - id: conversationId, - title, - messages, - replacements, - apiConfig, - }), - headers: { - 'Content-Type': 'application/json', - }, - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - signal, - }); + const response = await http.fetch( + `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${conversationId}`, + { + method: 'PUT', + body: JSON.stringify({ + id: conversationId, + title, + messages, + replacements, + apiConfig, + }), + headers: { + 'Content-Type': 'application/json', + }, + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + signal, + } + ); return response as Conversation; } catch (error) { diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/post_knowledge_base.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/index.tsx similarity index 53% rename from x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/post_knowledge_base.ts rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/index.tsx index 60d7f1064cefd..606574b12233b 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/post_knowledge_base.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/index.tsx @@ -5,9 +5,6 @@ * 2.0. */ -import * as t from 'io-ts'; - -/** Validates the URL path of a POST request to the `/knowledge_base/{resource}` endpoint */ -export const PostKnowledgeBasePathParams = t.type({ - resource: t.union([t.string, t.undefined]), -}); +export * from './conversations'; +export * from './use_bulk_actions_conversations'; +export * from './use_fetch_current_user_conversations'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts similarity index 95% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index fe04a318c8a4a..c1dcb78458e88 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -7,7 +7,7 @@ // import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; import { HttpSetup } from '@kbn/core/public'; -import { Conversation } from '../../assistant_context/types'; +import { Conversation } from '../../../assistant_context/types'; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY = 'elastic_assistant_conversations'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts similarity index 96% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts index 3fcacc48e883d..9d9670622ca73 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts @@ -13,7 +13,7 @@ import { ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; -import { Conversation } from '../../assistant_context/types'; +import { Conversation } from '../../../assistant_context/types'; export interface FetchConversationsResponse { page: number; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index be2e8a9e37289..55d095ca54c74 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -18,6 +18,8 @@ import { } from '../helpers'; import { PerformEvaluationParams } from '../settings/evaluation_settings/use_perform_evaluation'; +export * from './conversations'; + export interface FetchConnectorExecuteAction { isEnabledRAGAlerts: boolean; alertsIndexPattern?: string; @@ -122,6 +124,7 @@ export const fetchConnectorExecuteAction = async ({ signal, asResponse: isStream, rawResponse: isStream, + version: '1', } ); @@ -159,6 +162,7 @@ export const fetchConnectorExecuteAction = async ({ body: JSON.stringify(requestBody), headers: { 'Content-Type': 'application/json' }, signal, + version: '1', }); if (response.status !== 'ok' || !response.data) { @@ -252,6 +256,7 @@ export const getKnowledgeBaseStatus = async ({ const response = await http.fetch(path, { method: 'GET', signal, + version: '1', }); return response as GetKnowledgeBaseStatusResponse; @@ -290,6 +295,7 @@ export const postKnowledgeBase = async ({ const response = await http.fetch(path, { method: 'POST', signal, + version: '1', }); return response as PostKnowledgeBaseResponse; @@ -328,6 +334,7 @@ export const deleteKnowledgeBase = async ({ const response = await http.fetch(path, { method: 'DELETE', signal, + version: '1', }); return response as DeleteKnowledgeBaseResponse; @@ -386,6 +393,7 @@ export const postEvaluation = async ({ }, query, signal, + version: '1', }); return response as PostEvaluationResponse; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx index 427917002562d..cb075e26f4df4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx @@ -14,7 +14,7 @@ import styled from 'styled-components'; import { ShowAssistantOverlayProps, useAssistantContext } from '../../assistant_context'; import { Assistant } from '..'; import { WELCOME_CONVERSATION_TITLE } from '../use_conversation/translations'; -import { useFetchCurrentUserConversations } from '../api/use_fetch_current_user_conversations'; +import { useFetchCurrentUserConversations } from '../api/conversations/use_fetch_current_user_conversations'; const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 606931979173e..ce98ace90fa5e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -53,7 +53,7 @@ import { ConnectorMissingCallout } from '../connectorland/connector_missing_call import { useFetchCurrentUserConversations, useLastConversation, -} from './api/use_fetch_current_user_conversations'; +} from './api/conversations/use_fetch_current_user_conversations'; import { Conversation } from '../assistant_context/types'; export interface Props { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 7b07895f2a3e9..4a2b03b91a090 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -10,7 +10,7 @@ import { merge } from 'lodash'; import { Conversation, Prompt, QuickPrompt, useFetchCurrentUserConversations } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; -import { bulkConversationsChange } from '../../api/use_bulk_actions_conversations'; +import { bulkConversationsChange } from '../../api/conversations/use_bulk_actions_conversations'; interface UseSettingsUpdater { conversationSettings: Record; diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 5e69f84b65698..3aa191fa478d5 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -141,4 +141,4 @@ export type { DeleteKnowledgeBaseResponse } from './impl/assistant/api'; export type { GetKnowledgeBaseStatusResponse } from './impl/assistant/api'; export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; -export { useFetchCurrentUserConversations } from './impl/assistant/api/use_fetch_current_user_conversations'; +export { useFetchCurrentUserConversations } from './impl/assistant/api/conversations/use_fetch_current_user_conversations'; diff --git a/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts b/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts index 7d3c6e51e844e..01a1984187d43 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts @@ -147,6 +147,7 @@ export abstract class SubActionConnector { return res; } catch (error) { + console.log('hdfjskhfjksdhfjskdhfsjkdfhsjkdfhsjkdfhsjkdfhsjkdfhjkdshfdksjfhsdjfhsdxjk') if (isAxiosError(error)) { this.logger.debug( `Request to external service failed. Connector Id: ${this.connector.id}. Connector type: ${this.connector.type}. Method: ${error.config?.method}. URL: ${error.config?.url}` diff --git a/x-pack/plugins/elastic_assistant/common/constants.ts b/x-pack/plugins/elastic_assistant/common/constants.ts index 5770ffa6e17e0..fb3268211a922 100755 --- a/x-pack/plugins/elastic_assistant/common/constants.ts +++ b/x-pack/plugins/elastic_assistant/common/constants.ts @@ -21,5 +21,8 @@ export const EVALUATE = `${BASE_PATH}/evaluate`; export const MAX_CONVERSATIONS_TO_UPDATE_IN_PARALLEL = 50; export const CONVERSATIONS_TABLE_MAX_PAGE_SIZE = 100; +export const MAX_ANONIMIZATION_FIELDS_TO_UPDATE_IN_PARALLEL = 50; +export const ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE = 100; + // Capabilities export const CAPABILITIES = `${BASE_PATH}/capabilities`; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts similarity index 98% rename from x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts rename to x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts index 8131ec8514959..29e499cd594e2 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/conversation_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts @@ -6,7 +6,7 @@ */ import type { FieldMap } from '@kbn/alerts-as-data-utils'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; -import { IIndexPatternString } from '../../types'; +import { IIndexPatternString } from '../types'; export const conversationsFieldMap: FieldMap = { '@timestamp': { diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 69fcb5b0ba883..ef5f0b49e3a8d 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -20,7 +20,7 @@ import { errorResult, successResult, } from './create_resource_installation_helper'; -import { conversationsFieldMap } from './lib/conversation_configuration_type'; +import { conversationsFieldMap } from './conversation_configuration_type'; const TOTAL_FIELDS_LIMIT = 2500; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts deleted file mode 100644 index 689e81a5cc08e..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { loggingSystemMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { getRiskInputsIndex } from './get_risk_inputs_index'; -import { buildDataViewResponseMock } from './get_risk_inputs_index.mock'; - -describe('getRiskInputsIndex', () => { - let soClient: SavedObjectsClientContract; - let logger: Logger; - - beforeEach(() => { - soClient = savedObjectsClientMock.create(); - logger = loggingSystemMock.create().get('security_solution'); - }); - - it('returns an index and runtimeMappings for an existing dataView', async () => { - (soClient.get as jest.Mock).mockResolvedValueOnce(buildDataViewResponseMock()); - const { - id, - attributes: { runtimeFieldMap, title }, - } = buildDataViewResponseMock(); - - const { index, runtimeMappings } = await getRiskInputsIndex({ - dataViewId: id, - logger, - soClient, - }); - - expect(index).toEqual(title); - expect(runtimeMappings).toEqual(JSON.parse(runtimeFieldMap as string)); - }); - - it('returns the index and empty runtimeMappings for a nonexistent dataView', async () => { - const { index, runtimeMappings } = await getRiskInputsIndex({ - dataViewId: 'my-data-view', - logger, - soClient, - }); - expect(index).toEqual('my-data-view'); - expect(runtimeMappings).toEqual({}); - }); - - it('logs that the dataview was not found', async () => { - await getRiskInputsIndex({ - dataViewId: 'my-data-view', - logger, - soClient, - }); - expect(logger.info).toHaveBeenCalledWith( - "No dataview found for ID 'my-data-view'; using ID instead as simple index pattern" - ); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts deleted file mode 100644 index cfd66e63159a2..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/lib/get_configuration_index.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; -import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import type { DataViewAttributes } from '@kbn/data-views-plugin/common'; - -export interface ConfigurationResponse { - index: string; - runtimeMappings: MappingRuntimeFields; -} - -export const getConfigurationIndex = async ({ - dataViewId, - logger, - soClient, -}: { - dataViewId: string; - logger: Logger; - soClient: SavedObjectsClientContract; -}): Promise => { - try { - const dataView = await soClient.get('index-pattern', dataViewId); - const index = dataView.attributes.title; - const runtimeMappings = - dataView.attributes.runtimeFieldMap != null - ? JSON.parse(dataView.attributes.runtimeFieldMap) - : {}; - - return { - index, - runtimeMappings, - }; - } catch (e) { - logger.info( - `No dataview found for ID '${dataViewId}'; using ID instead as simple index pattern` - ); - return { index: dataViewId, runtimeMappings: {} }; - } -}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 2a23071eb0793..9f35548cd03cb 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -13,7 +13,7 @@ import { AuthenticatedUser } from '@kbn/security-plugin/server'; import { estypes } from '@elastic/elasticsearch'; import { IIndexPatternString } from '../types'; import { ConversationDataWriter } from './conversations_data_writer'; -import { getIndexTemplateAndPattern } from '../ai_assistant_service/lib/conversation_configuration_type'; +import { getIndexTemplateAndPattern } from '../ai_assistant_service/conversation_configuration_type'; import { createConversation } from './create_conversation'; import { ConversationCreateProps, diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts new file mode 100644 index 0000000000000..8c8f59ff612ee --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; + +import { SavedObjectError } from '@kbn/core/types'; +import { ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { + AnonimizationFieldResponse, + BulkActionSkipResult, + BulkCrudActionResponse, + BulkCrudActionResults, + BulkCrudActionSummary, + PerformBulkActionRequestBody, + PerformBulkActionResponse, +} from '../../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; + +export interface BulkOperationError { + message: string; + status?: number; + anonimizationField: { + id: string; + name: string; + }; +} + +export type BulkActionError = BulkOperationError | unknown; + +const buildBulkResponse = ( + response: KibanaResponseFactory, + { + errors = [], + updated = [], + created = [], + deleted = [], + skipped = [], + }: { + errors?: SavedObjectError[]; + updated?: AnonimizationFieldResponse[]; + created?: AnonimizationFieldResponse[]; + deleted?: string[]; + skipped?: BulkActionSkipResult[]; + } +): IKibanaResponse => { + const numSucceeded = updated.length + created.length + deleted.length; + const numSkipped = skipped.length; + const numFailed = errors.length; + + const summary: BulkCrudActionSummary = { + failed: numFailed, + succeeded: numSucceeded, + skipped: numSkipped, + total: numSucceeded + numFailed + numSkipped, + }; + + const results: BulkCrudActionResults = { + updated, + created, + deleted, + skipped, + }; + + if (numFailed > 0) { + return response.custom({ + headers: { 'content-type': 'application/json' }, + body: { + message: summary.succeeded > 0 ? 'Bulk edit partially failed' : 'Bulk edit failed', + status_code: 500, + attributes: { + errors: [], + results, + summary, + }, + }, + statusCode: 500, + }); + } + + const responseBody: BulkCrudActionResponse = { + success: true, + anonimization_fields_count: summary.total, + attributes: { results, summary }, + }; + + return response.ok({ body: responseBody }); +}; + +export const bulkActionAnonimizationFieldsRoute = ( + router: ElasticAssistantPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION, + options: { + tags: ['access:elasticAssistant'], + timeout: { + idleSocket: moment.duration(15, 'minutes').asMilliseconds(), + }, + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PerformBulkActionRequestBody), + }, + }, + }, + async (context, request, response): Promise> => { + const { body } = request; + const assistantResponse = buildResponse(response); + + if (body?.update && body.update?.length > ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE) { + return assistantResponse.error({ + body: `More than ${ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, + statusCode: 400, + }); + } + + const abortController = new AbortController(); + + // subscribing to completed$, because it handles both cases when request was completed and aborted. + // when route is finished by timeout, aborted$ is not getting fired + request.events.completed$.subscribe(() => abortController.abort()); + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantAnonimizationFieldsSOClient(); + + const docsCreated = + body.create && body.create.length > 0 + ? await dataClient.createAnonimizationFields(body.create) + : []; + const docsUpdated = + body.update && body.update.length > 0 + ? await dataClient.updateAnonimizationFields(body.update) + : []; + const docsDeleted = await dataClient.deleteAnonimizationFieldsByIds( + body.delete?.ids ?? [] + ); + + const created = await dataClient?.findAnonimizationFields({ + page: 1, + perPage: 1000, + filter: docsCreated.map((updatedId) => `id:${updatedId}`).join(' OR '), + fields: ['id'], + }); + const updated = await dataClient?.findAnonimizationFields({ + page: 1, + perPage: 1000, + filter: docsUpdated.map((updatedId) => `id:${updatedId}`).join(' OR '), + fields: ['id'], + }); + + return buildBulkResponse(response, { + updated: updated?.data, + created: created?.data, + deleted: docsDeleted.map((d) => d.id) ?? [], + errors: docsDeleted.reduce((res, d) => { + if (d.error !== undefined) { + res.push(d.error); + } + return res; + }, [] as SavedObjectError[]), + }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts new file mode 100644 index 0000000000000..1d6e7025c11de --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { + FindAnonimizationFieldsRequestQuery, + FindAnonimizationFieldsResponse, +} from '../../schemas/anonimization_fields/find_prompts_route.gen'; + +export const findAnonimizationFieldsRoute = ( + router: ElasticAssistantPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + query: buildRouteValidationWithZod(FindAnonimizationFieldsRequestQuery), + }, + }, + }, + async ( + context, + request, + response + ): Promise> => { + const assistantResponse = buildResponse(response); + + try { + const { query } = request; + const ctx = await context.resolve(['core', 'elasticAssistant']); + const dataClient = await ctx.elasticAssistant.getAIAssistantAnonimizationFieldsSOClient(); + + const result = await dataClient?.findAnonimizationFields({ + perPage: query.per_page, + page: query.page, + sortField: query.sort_field, + sortOrder: query.sort_order, + filter: query.filter, + fields: query.fields, + }); + + return response.ok({ body: result }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index 332d44196c4af..b2cf67291704d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -9,6 +9,7 @@ import { IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { v4 as uuidv4 } from 'uuid'; +import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; @@ -49,200 +50,211 @@ export const postEvaluateRoute = ( router: IRouter, getElser: GetElser ) => { - router.post( - { + router.versioned + .post({ + access: 'internal', path: EVALUATE, - validate: { - body: buildRouteValidationWithZod(EvaluateRequestBody), - query: buildRouteValidationWithZod(EvaluateRequestQuery), + + options: { + tags: ['access:elasticAssistant'], }, - }, - async (context, request, response) => { - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; - - // Validate evaluation feature is enabled - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); - if (!registeredFeatures.assistantModelEvaluation) { - return response.notFound(); - } + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(EvaluateRequestBody), + query: buildRouteValidationWithZod(EvaluateRequestQuery), + }, + }, + }, + async (context, request, response) => { + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; - try { - const evaluationId = uuidv4(); - const { - evalModel, - evaluationType, - outputIndex, - datasetName, - projectName = 'default', - runName = evaluationId, - } = request.query; - const { dataset: customDataset = [], evalPrompt } = request.body; - const connectorIds = request.query.models?.split(',') || []; - const agentNames = request.query.agents?.split(',') || []; - - const dataset = - datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; - - logger.info('postEvaluateRoute:'); - logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); - logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); - logger.info(`Evaluation ID: ${evaluationId}`); - - const totalExecutions = connectorIds.length * agentNames.length * dataset.length; - logger.info('Creating agents:'); - logger.info(`\tconnectors/models: ${connectorIds.length}`); - logger.info(`\tagents: ${agentNames.length}`); - logger.info(`\tdataset: ${dataset.length}`); - logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); - if (totalExecutions > 50) { - logger.warn( - `Total baseline agent executions >= 50! This may take a while, and cost some money...` - ); + // Validate evaluation feature is enabled + const pluginName = getPluginNameFromRequest({ + request, + defaultPluginName: DEFAULT_PLUGIN_NAME, + logger, + }); + const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName); + if (!registeredFeatures.assistantModelEvaluation) { + return response.notFound(); } - // Get the actions plugin start contract from the request context for the agents - const actions = (await context.elasticAssistant).actions; + try { + const evaluationId = uuidv4(); + const { + evalModel, + evaluationType, + outputIndex, + datasetName, + projectName = 'default', + runName = evaluationId, + } = request.query; + const { dataset: customDataset = [], evalPrompt } = request.body; + const connectorIds = request.query.models?.split(',') || []; + const agentNames = request.query.agents?.split(',') || []; - // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm - const actionsClient = await actions.getActionsClientWithRequest(request); - const connectors = await actionsClient.getBulk({ - ids: connectorIds, - throwIfSystemAction: false, - }); + const dataset = + datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset; + + logger.info('postEvaluateRoute:'); + logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`); + logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`); + logger.info(`Evaluation ID: ${evaluationId}`); + + const totalExecutions = connectorIds.length * agentNames.length * dataset.length; + logger.info('Creating agents:'); + logger.info(`\tconnectors/models: ${connectorIds.length}`); + logger.info(`\tagents: ${agentNames.length}`); + logger.info(`\tdataset: ${dataset.length}`); + logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `); + if (totalExecutions > 50) { + logger.warn( + `Total baseline agent executions >= 50! This may take a while, and cost some money...` + ); + } - // Fetch any tools registered by the request's originating plugin - const assistantTools = (await context.elasticAssistant).getRegisteredTools( - 'securitySolution' - ); - - // Get a scoped esClient for passing to the agents for retrieval, and - // writing results to the output index - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - - // Default ELSER model - const elserId = await getElser(request, (await context.core).savedObjects.getClient()); - - // Skeleton request from route to pass to the agents - // params will be passed to the actions executor - const skeletonRequest: KibanaRequest = { - ...request, - body: { - alertsIndexPattern: '', - allow: [], - allowReplacement: [], - params: { - subAction: 'invokeAI', - subActionParams: { - messages: [], + // Get the actions plugin start contract from the request context for the agents + const actions = (await context.elasticAssistant).actions; + + // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm + const actionsClient = await actions.getActionsClientWithRequest(request); + const connectors = await actionsClient.getBulk({ + ids: connectorIds, + throwIfSystemAction: false, + }); + + // Fetch any tools registered by the request's originating plugin + const assistantTools = (await context.elasticAssistant).getRegisteredTools( + 'securitySolution' + ); + + // Get a scoped esClient for passing to the agents for retrieval, and + // writing results to the output index + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + // Default ELSER model + const elserId = await getElser(request, (await context.core).savedObjects.getClient()); + + // Skeleton request from route to pass to the agents + // params will be passed to the actions executor + const skeletonRequest: KibanaRequest = { + ...request, + body: { + alertsIndexPattern: '', + allow: [], + allowReplacement: [], + params: { + subAction: 'invokeAI', + subActionParams: { + messages: [], + }, }, + replacements: {}, + size: DEFAULT_SIZE, + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: true, }, - replacements: {}, - size: DEFAULT_SIZE, - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: true, - }, - }; - - // Create an array of executor functions to call in batches - // One for each connector/model + agent combination - // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator - const agents: AgentExecutorEvaluatorWithMetadata[] = []; - connectorIds.forEach((connectorId) => { - agentNames.forEach((agentName) => { - logger.info(`Creating agent: ${connectorId} + ${agentName}`); - const llmType = getLlmType(connectorId, connectors); - const connectorName = - getConnectorName(connectorId, connectors) ?? '[unknown connector]'; - const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; - agents.push({ - agentEvaluator: (langChainMessages, exampleId) => - AGENT_EXECUTOR_MAP[agentName]({ + }; + + // Create an array of executor functions to call in batches + // One for each connector/model + agent combination + // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator + const agents: AgentExecutorEvaluatorWithMetadata[] = []; + connectorIds.forEach((connectorId) => { + agentNames.forEach((agentName) => { + logger.info(`Creating agent: ${connectorId} + ${agentName}`); + const llmType = getLlmType(connectorId, connectors); + const connectorName = + getConnectorName(connectorId, connectors) ?? '[unknown connector]'; + const detailedRunName = `${runName} - ${connectorName} + ${agentName}`; + agents.push({ + agentEvaluator: (langChainMessages, exampleId) => + AGENT_EXECUTOR_MAP[agentName]({ + actions, + isEnabledKnowledgeBase: true, + assistantTools, + connectorId, + esClient, + elserId, + langChainMessages, + llmType, + logger, + request: skeletonRequest, + kbResource: ESQL_RESOURCE, + telemetry, + traceOptions: { + exampleId, + projectName, + runName: detailedRunName, + evaluationId, + tags: [ + 'security-assistant-prediction', + ...(connectorName != null ? [connectorName] : []), + runName, + ], + tracers: getLangSmithTracer(detailedRunName, exampleId, logger), + }, + }), + metadata: { + connectorName, + runName: detailedRunName, + }, + }); + }); + }); + logger.info(`Agents created: ${agents.length}`); + + // Evaluator Model is optional to support just running predictions + const evaluatorModel = + evalModel == null || evalModel === '' + ? undefined + : new ActionsClientLlm({ actions, - isEnabledKnowledgeBase: true, - assistantTools, - connectorId, - esClient, - elserId, - langChainMessages, - llmType, - logger, + connectorId: evalModel, request: skeletonRequest, - kbResource: ESQL_RESOURCE, - telemetry, - traceOptions: { - exampleId, - projectName, - runName: detailedRunName, - evaluationId, - tags: [ - 'security-assistant-prediction', - ...(connectorName != null ? [connectorName] : []), - runName, - ], - tracers: getLangSmithTracer(detailedRunName, exampleId, logger), - }, - }), - metadata: { - connectorName, - runName: detailedRunName, - }, - }); + logger, + }); + + const { evaluationResults, evaluationSummary } = await performEvaluation({ + agentExecutorEvaluators: agents, + dataset, + evaluationId, + evaluatorModel, + evaluationPrompt: evalPrompt, + evaluationType, + logger, + runName, }); - }); - logger.info(`Agents created: ${agents.length}`); - - // Evaluator Model is optional to support just running predictions - const evaluatorModel = - evalModel == null || evalModel === '' - ? undefined - : new ActionsClientLlm({ - actions, - connectorId: evalModel, - request: skeletonRequest, - logger, - }); - const { evaluationResults, evaluationSummary } = await performEvaluation({ - agentExecutorEvaluators: agents, - dataset, - evaluationId, - evaluatorModel, - evaluationPrompt: evalPrompt, - evaluationType, - logger, - runName, - }); + logger.info(`Writing evaluation results to index: ${outputIndex}`); + await setupEvaluationIndex({ esClient, index: outputIndex, logger }); + await indexEvaluations({ + esClient, + evaluationResults, + evaluationSummary, + index: outputIndex, + logger, + }); - logger.info(`Writing evaluation results to index: ${outputIndex}`); - await setupEvaluationIndex({ esClient, index: outputIndex, logger }); - await indexEvaluations({ - esClient, - evaluationResults, - evaluationSummary, - index: outputIndex, - logger, - }); + return response.ok({ + body: { evaluationId, success: true }, + }); + } catch (err) { + logger.error(err); + const error = transformError(err); - return response.ok({ - body: { evaluationId, success: true }, - }); - } catch (err) { - logger.error(err); - const error = transformError(err); - - const resp = buildResponse(response); - return resp.error({ - body: { success: false, error: error.message }, - statusCode: error.statusCode, - }); + const resp = buildResponse(response); + return resp.error({ + body: { success: false, error: error.message }, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts index 235ea4eda4c0d..f6e91cd671332 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts @@ -5,17 +5,20 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; +import { IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import type { DeleteKnowledgeBaseResponse } from '@kbn/elastic-assistant'; +import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; import { buildResponse } from '../../lib/build_response'; -import { buildRouteValidation } from '../../schemas/common'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { KNOWLEDGE_BASE } from '../../../common/constants'; import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elasticsearch_store'; import { ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; -import { DeleteKnowledgeBasePathParams } from '../../schemas/knowledge_base/delete_knowledge_base'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { + DeleteKnowledgeBaseRequestParams, + DeleteKnowledgeBaseResponse, +} from '../../schemas/knowledge_base/crud_kb_route.gen'; /** * Delete Knowledge Base index, pipeline, and resources (collection of documents) @@ -24,63 +27,72 @@ import { DeleteKnowledgeBasePathParams } from '../../schemas/knowledge_base/dele export const deleteKnowledgeBaseRoute = ( router: IRouter ) => { - router.delete( - { + router.versioned + .delete({ + access: 'internal', path: KNOWLEDGE_BASE, - validate: { - params: buildRouteValidation(DeleteKnowledgeBasePathParams), - }, options: { // Note: Relying on current user privileges to scope an esClient. // Add `access:kbnElasticAssistant` to limit API access to only users with assistant privileges tags: [], }, - }, - async (context, request, response) => { - const resp = buildResponse(response); - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(DeleteKnowledgeBaseRequestParams), + }, + }, + }, + async (context, request: KibanaRequest, response) => { + const resp = buildResponse(response); + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; - try { - const kbResource = - request.params.resource != null ? decodeURIComponent(request.params.resource) : undefined; + try { + const kbResource = + request.params.resource != null + ? decodeURIComponent(request.params.resource) + : undefined; - // Get a scoped esClient for deleting the Knowledge Base index, pipeline, and documents - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const esStore = new ElasticsearchStore( - esClient, - KNOWLEDGE_BASE_INDEX_PATTERN, - logger, - telemetry - ); + // Get a scoped esClient for deleting the Knowledge Base index, pipeline, and documents + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const esStore = new ElasticsearchStore( + esClient, + KNOWLEDGE_BASE_INDEX_PATTERN, + logger, + telemetry + ); - if (kbResource === ESQL_RESOURCE) { - // For now, tearing down the Knowledge Base is fine, but will want to support removing specific assets based - // on resource name or document query - // Implement deleteDocuments(query: string) in ElasticsearchStore - // const success = await esStore.deleteDocuments(); - // return const body: DeleteKnowledgeBaseResponse = { success }; - } + if (kbResource === ESQL_RESOURCE) { + // For now, tearing down the Knowledge Base is fine, but will want to support removing specific assets based + // on resource name or document query + // Implement deleteDocuments(query: string) in ElasticsearchStore + // const success = await esStore.deleteDocuments(); + // return const body: DeleteKnowledgeBaseResponse = { success }; + } - // Delete index and pipeline - const indexDeleted = await esStore.deleteIndex(); - const pipelineDeleted = await esStore.deletePipeline(); + // Delete index and pipeline + const indexDeleted = await esStore.deleteIndex(); + const pipelineDeleted = await esStore.deletePipeline(); - const body: DeleteKnowledgeBaseResponse = { - success: indexDeleted && pipelineDeleted, - }; + const body: DeleteKnowledgeBaseResponse = { + success: indexDeleted && pipelineDeleted, + }; - return response.ok({ body }); - } catch (err) { - logger.error(err); - const error = transformError(err); + return response.ok({ body }); + } catch (err) { + logger.error(err); + const error = transformError(err); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts index a238a8f55d615..df3e112e1704c 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts @@ -5,14 +5,8 @@ * 2.0. */ -/** - * A knowledge base REST request - */ -interface KbRequest { - params?: { - resource?: string; - }; -} +import { KibanaRequest } from '@kbn/core/server'; +import { CreateKnowledgeBaseRequestParams } from '../../schemas/knowledge_base/crud_kb_route.gen'; /** * Returns the optional resource, e.g. `esql` from the request params, or undefined if it doesn't exist @@ -21,7 +15,9 @@ interface KbRequest { * * @returns Returns the optional resource, e.g. `esql` from the request params, or undefined if it doesn't exist */ -export const getKbResource = (request: KbRequest | undefined): string | undefined => { +export const getKbResource = ( + request: KibanaRequest | undefined +): string | undefined => { if (request?.params?.resource != null) { return decodeURIComponent(request.params.resource); } else { diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts index c61887a436267..924339c698b1a 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts @@ -5,18 +5,21 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import type { GetKnowledgeBaseStatusResponse } from '@kbn/elastic-assistant'; +import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { KibanaRequest } from '@kbn/core/server'; import { getKbResource } from './get_kb_resource'; import { buildResponse } from '../../lib/build_response'; -import { buildRouteValidation } from '../../schemas/common'; -import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; +import { ElasticAssistantPluginRouter, GetElser } from '../../types'; import { KNOWLEDGE_BASE } from '../../../common/constants'; -import { GetKnowledgeBaseStatusPathParams } from '../../schemas/knowledge_base/get_knowledge_base_status'; import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elasticsearch_store'; import { ESQL_DOCS_LOADED_QUERY, ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; +import { + ReadKnowledgeBaseRequestParams, + ReadKnowledgeBaseResponse, +} from '../../schemas/knowledge_base/crud_kb_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; /** * Get the status of the Knowledge Base index, pipeline, and resources (collection of documents) @@ -25,67 +28,74 @@ import { ESQL_DOCS_LOADED_QUERY, ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } f * @param getElser Function to get the default Elser ID */ export const getKnowledgeBaseStatusRoute = ( - router: IRouter, + router: ElasticAssistantPluginRouter, getElser: GetElser ) => { - router.get( - { + router.versioned + .get({ + access: 'internal', path: KNOWLEDGE_BASE, - validate: { - params: buildRouteValidation(GetKnowledgeBaseStatusPathParams), - }, options: { // Note: Relying on current user privileges to scope an esClient. // Add `access:kbnElasticAssistant` to limit API access to only users with assistant privileges tags: [], }, - }, - async (context, request, response) => { - const resp = buildResponse(response); - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(ReadKnowledgeBaseRequestParams), + }, + }, + }, + async (context, request: KibanaRequest, response) => { + const resp = buildResponse(response); + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; - try { - // Get a scoped esClient for finding the status of the Knowledge Base index, pipeline, and documents - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const elserId = await getElser(request, (await context.core).savedObjects.getClient()); - const kbResource = getKbResource(request); - const esStore = new ElasticsearchStore( - esClient, - KNOWLEDGE_BASE_INDEX_PATTERN, - logger, - telemetry, - elserId, - kbResource - ); + try { + // Get a scoped esClient for finding the status of the Knowledge Base index, pipeline, and documents + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const elserId = await getElser(request, (await context.core).savedObjects.getClient()); + const kbResource = getKbResource(request); + const esStore = new ElasticsearchStore( + esClient, + KNOWLEDGE_BASE_INDEX_PATTERN, + logger, + telemetry, + elserId, + kbResource + ); - const indexExists = await esStore.indexExists(); - const pipelineExists = await esStore.pipelineExists(); - const modelExists = await esStore.isModelInstalled(elserId); + const indexExists = await esStore.indexExists(); + const pipelineExists = await esStore.pipelineExists(); + const modelExists = await esStore.isModelInstalled(elserId); - const body: GetKnowledgeBaseStatusResponse = { - elser_exists: modelExists, - index_exists: indexExists, - pipeline_exists: pipelineExists, - }; + const body: ReadKnowledgeBaseResponse = { + elser_exists: modelExists, + index_exists: indexExists, + pipeline_exists: pipelineExists, + }; - if (kbResource === ESQL_RESOURCE) { - const esqlExists = - indexExists && (await esStore.similaritySearch(ESQL_DOCS_LOADED_QUERY)).length > 0; - return response.ok({ body: { ...body, esql_exists: esqlExists } }); - } + if (kbResource === ESQL_RESOURCE) { + const esqlExists = + indexExists && (await esStore.similaritySearch(ESQL_DOCS_LOADED_QUERY)).length > 0; + return response.ok({ body: { ...body, esql_exists: esqlExists } }); + } - return response.ok({ body }); - } catch (err) { - logger.error(err); - const error = transformError(err); + return response.ok({ body }); + } catch (err) { + logger.error(err); + const error = transformError(err); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts index 56812ee8d0305..a9f908117aef8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts @@ -5,99 +5,114 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; +import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { IKibanaResponse, KibanaRequest } from '@kbn/core/server'; import { buildResponse } from '../../lib/build_response'; -import { buildRouteValidation } from '../../schemas/common'; -import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; +import { ElasticAssistantPluginRouter, GetElser } from '../../types'; import { KNOWLEDGE_BASE } from '../../../common/constants'; import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elasticsearch_store'; import { ESQL_DOCS_LOADED_QUERY, ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; import { getKbResource } from './get_kb_resource'; -import { PostKnowledgeBasePathParams } from '../../schemas/knowledge_base/post_knowledge_base'; import { loadESQL } from '../../lib/langchain/content_loaders/esql_loader'; +import { + CreateKnowledgeBaseRequestParams, + CreateKnowledgeBaseResponse, +} from '../../schemas/knowledge_base/crud_kb_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; /** * Load Knowledge Base index, pipeline, and resources (collection of documents) * @param router */ export const postKnowledgeBaseRoute = ( - router: IRouter, + router: ElasticAssistantPluginRouter, getElser: GetElser ) => { - router.post( - { + router.versioned + .post({ + access: 'internal', path: KNOWLEDGE_BASE, - validate: { - params: buildRouteValidation(PostKnowledgeBasePathParams), - }, options: { // Note: Relying on current user privileges to scope an esClient. // Add `access:kbnElasticAssistant` to limit API access to only users with assistant privileges tags: [], }, - }, - async (context, request, response) => { - const resp = buildResponse(response); - const assistantContext = await context.elasticAssistant; - const logger = assistantContext.logger; - const telemetry = assistantContext.telemetry; + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(CreateKnowledgeBaseRequestParams), + }, + }, + }, + async ( + context, + request: KibanaRequest, + response + ): Promise> => { + const resp = buildResponse(response); + const assistantContext = await context.elasticAssistant; + const logger = assistantContext.logger; + const telemetry = assistantContext.telemetry; - try { - const core = await context.core; - // Get a scoped esClient for creating the Knowledge Base index, pipeline, and documents - const esClient = core.elasticsearch.client.asCurrentUser; - const elserId = await getElser(request, core.savedObjects.getClient()); - const kbResource = getKbResource(request); - const esStore = new ElasticsearchStore( - esClient, - KNOWLEDGE_BASE_INDEX_PATTERN, - logger, - telemetry, - elserId, - kbResource - ); + try { + const core = await context.core; + // Get a scoped esClient for creating the Knowledge Base index, pipeline, and documents + const esClient = core.elasticsearch.client.asCurrentUser; + const elserId = await getElser(request, core.savedObjects.getClient()); + const kbResource = getKbResource(request); + const esStore = new ElasticsearchStore( + esClient, + KNOWLEDGE_BASE_INDEX_PATTERN, + logger, + telemetry, + elserId, + kbResource + ); - // Pre-check on index/pipeline - let indexExists = await esStore.indexExists(); - let pipelineExists = await esStore.pipelineExists(); + // Pre-check on index/pipeline + let indexExists = await esStore.indexExists(); + let pipelineExists = await esStore.pipelineExists(); - // Load if not exists - if (!pipelineExists) { - pipelineExists = await esStore.createPipeline(); - } - if (!indexExists) { - indexExists = await esStore.createIndex(); - } + // Load if not exists + if (!pipelineExists) { + pipelineExists = await esStore.createPipeline(); + } + if (!indexExists) { + indexExists = await esStore.createIndex(); + } - // If specific resource is requested, load it - if (kbResource === ESQL_RESOURCE) { - const esqlExists = (await esStore.similaritySearch(ESQL_DOCS_LOADED_QUERY)).length > 0; - if (!esqlExists) { - const loadedKnowledgeBase = await loadESQL(esStore, logger); - return response.custom({ body: { success: loadedKnowledgeBase }, statusCode: 201 }); - } else { - return response.ok({ body: { success: true } }); + // If specific resource is requested, load it + if (kbResource === ESQL_RESOURCE) { + const esqlExists = (await esStore.similaritySearch(ESQL_DOCS_LOADED_QUERY)).length > 0; + if (!esqlExists) { + const loadedKnowledgeBase = await loadESQL(esStore, logger); + return response.custom({ body: { success: loadedKnowledgeBase }, statusCode: 201 }); + } else { + return response.ok({ body: { success: true } }); + } } - } - const wasSuccessful = indexExists && pipelineExists; + const wasSuccessful = indexExists && pipelineExists; - if (wasSuccessful) { - return response.ok({ body: { success: true } }); - } else { - return response.custom({ body: { success: false }, statusCode: 500 }); - } - } catch (err) { - logger.log(err); - const error = transformError(err); + if (wasSuccessful) { + return response.ok({ body: { success: true } }); + } else { + return response.custom({ body: { success: false }, statusCode: 500 }); + } + } catch (err) { + logger.log(err); + const error = transformError(err); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index f86236d52d298..fe8b48435d77d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -9,7 +9,7 @@ import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { schema } from '@kbn/config-schema'; -import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION } from '@kbn/elastic-assistant-common'; +import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; import { INVOKE_ASSISTANT_ERROR_EVENT, INVOKE_ASSISTANT_SUCCESS_EVENT, @@ -43,7 +43,7 @@ export const postActionsConnectorExecuteRoute = ( }) .addVersion( { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, validate: { request: { body: buildRouteValidationWithZod(ExecuteConnectorRequestBody), diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index bf6b9ee6cc5f7..92ff2cf4e1fa9 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -19,6 +19,7 @@ import { import { AIAssistantPromptsSOClient } from '../saved_object/ai_assistant_prompts_so_client'; import { AIAssistantService } from '../ai_assistant_service'; import { appContextService } from '../services/app_context'; +import { AIAssistantAnonimizationFieldsSOClient } from '../saved_object/ai_assistant_anonimization_fields_so_client'; export interface IRequestContextFactory { create( @@ -92,6 +93,16 @@ export class RequestContextFactory implements IRequestContextFactory { }); }), + getAIAssistantAnonimizationFieldsSOClient: memoize(() => { + const username = + startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; + return new AIAssistantAnonimizationFieldsSOClient({ + logger: options.logger, + user: username, + savedObjectsClient: coreContext.savedObjects.client, + }); + }), + getAIAssistantConversationsDataClient: memoize(async () => { const currentUser = getCurrentUser(); return this.assistantService.createAIAssistantDatastreamClient({ diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts new file mode 100644 index 0000000000000..4584c2179398e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Logger, + SavedObjectsErrorHelpers, + type SavedObjectsClientContract, + SavedObjectsBulkDeleteStatus, +} from '@kbn/core/server'; + +import { + AssistantAnonimizationFieldSoSchema, + assistantAnonimizationFieldsTypeName, + transformSavedObjectToAssistantAnonimizationField, + transformSavedObjectUpdateToAssistantAnonimizationField, + transformSavedObjectsToFoundAssistantAnonimizationField, +} from './elastic_assistant_anonimization_fields_type'; +import { + AnonimizationFieldCreateProps, + AnonimizationFieldResponse, + AnonimizationFieldUpdateProps, +} from '../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; +import { + FindAnonimizationFieldsResponse, + SortOrder, +} from '../schemas/anonimization_fields/find_prompts_route.gen'; + +export interface ConstructorOptions { + /** User creating, modifying, deleting, or updating the anonimization fields */ + user: string; + /** Saved objects client to create, modify, delete, the anonimization fields */ + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +/** + * Class for use for anonimization fields that are used for AI assistant. + */ +export class AIAssistantAnonimizationFieldsSOClient { + /** User creating, modifying, deleting, or updating the anonimization fields */ + private readonly user: string; + + /** Saved objects client to create, modify, delete, the anonimization fields */ + private readonly savedObjectsClient: SavedObjectsClientContract; + + /** + * Constructs the assistant client + * @param options + * @param options.user The user associated with the anonimization fields + * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI anonimization fields + */ + constructor({ user, savedObjectsClient }: ConstructorOptions) { + this.user = user; + this.savedObjectsClient = savedObjectsClient; + } + + /** + * Fetch an anonimization field + * @param options + * @param options.id the "id" of an exception list + * @returns The found exception list or null if none exists + */ + public getAnonimizationField = async (id: string): Promise => { + const { savedObjectsClient } = this; + if (id != null) { + try { + const savedObject = await savedObjectsClient.get( + assistantAnonimizationFieldsTypeName, + id + ); + return transformSavedObjectToAssistantAnonimizationField({ savedObject }); + } catch (err) { + if (SavedObjectsErrorHelpers.isNotFoundError(err)) { + return null; + } else { + throw err; + } + } + } else { + return null; + } + }; + + /** + * This creates an agnostic space endpoint list if it does not exist. This tries to be + * as fast as possible by ignoring conflict errors and not returning the contents of the + * list if it already exists. + * @returns AssistantAnonimizationFieldSchema if it created the endpoint list, otherwise null if it already exists + */ + public createAnonimizationFields = async ( + items: AnonimizationFieldCreateProps[] + ): Promise => { + const { savedObjectsClient, user } = this; + + const dateNow = new Date().toISOString(); + try { + const formattedItems = items.map((item) => { + return { + attributes: { + created_at: dateNow, + created_by: user, + field_id: item.fieldId, + default_allow: item.defaultAllow ?? false, + default_allow_replacement: item.defaultAllowReplacement ?? false, + updated_by: user, + updated_at: dateNow, + }, + type: assistantAnonimizationFieldsTypeName, + }; + }); + const savedObjectsBulk = + await savedObjectsClient.bulkCreate(formattedItems); + + const result = savedObjectsBulk.saved_objects.map((savedObject) => + transformSavedObjectToAssistantAnonimizationField({ savedObject }) + ); + return result; + } catch (err) { + if (SavedObjectsErrorHelpers.isConflictError(err)) { + return []; + } else { + throw err; + } + } + }; + + /** + * This is the same as "updateExceptionListItem" except it applies specifically to the endpoint list and will + * auto-call the "createEndpointList" for you so that you have the best chance of the endpoint + * being there if it did not exist before. If the list did not exist before, then creating it here will still cause a + * return of null but at least the list exists again. + * @param options + * @param options._version The version to update the endpoint list item to + * @param options.comments The comments of the endpoint list item + * @param options.description The description of the endpoint list item + * @param options.entries The entries of the endpoint list item + * @param options.id The id of the list item (Either this or itemId has to be defined) + * @param options.itemId The item id of the list item (Either this or id has to be defined) + * @param options.meta Optional meta data of the list item + * @param options.name The name of the list item + * @param options.osTypes The OS type of the list item + * @param options.tags Tags of the endpoint list item + * @param options.type The type of the endpoint list item (Default is "simple") + * @returns The exception list item updated, otherwise null if not updated + */ + public updateAnonimizationFields = async ( + items: AnonimizationFieldUpdateProps[] + ): Promise => { + const { savedObjectsClient, user } = this; + const dateNow = new Date().toISOString(); + + const existingItems = ( + await this.findAnonimizationFields({ + page: 1, + perPage: 1000, + filter: items.map((updated) => `id:${updated.id}`).join(' OR '), + fields: ['id'], + }) + ).data.reduce((res, item) => { + res[item.id] = item; + return res; + }, {} as Record); + const formattedItems = items.map((item) => { + return { + attributes: { + default_allow: item.defaultAllow ?? false, + default_allow_replacement: item.defaultAllowReplacement ?? false, + updated_by: user, + updated_at: dateNow, + }, + id: existingItems[item.id].id, + type: assistantAnonimizationFieldsTypeName, + }; + }); + const savedObjectsBulk = + await savedObjectsClient.bulkUpdate(formattedItems); + const result = savedObjectsBulk.saved_objects.map((savedObject) => + transformSavedObjectUpdateToAssistantAnonimizationField({ savedObject }) + ); + return result; + }; + + /** + * Delete the anonimization field by id + * @param options + * @param options.id the "id" of the anonimization field + */ + public deleteAnonimizationFieldsByIds = async ( + ids: string[] + ): Promise => { + const { savedObjectsClient } = this; + + const res = await savedObjectsClient.bulkDelete( + ids.map((id) => ({ id, type: assistantAnonimizationFieldsTypeName })) + ); + return res.statuses; + }; + + /** + * Finds anonimization fields given a set of criteria. + * @param options + * @param options.filter The filter to apply in the search + * @param options.perPage How many per page to return + * @param options.page The page number or "undefined" if there is no page number to continue from + * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in + * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in + * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in + * @returns The found anonimization fields or null if nothing is found + */ + public findAnonimizationFields = async ({ + perPage, + page, + sortField, + sortOrder, + filter, + fields, + }: { + perPage: number; + page: number; + sortField?: string; + sortOrder?: SortOrder; + filter?: string; + fields?: string[]; + }): Promise => { + const { savedObjectsClient } = this; + + const savedObjectsFindResponse = + await savedObjectsClient.find({ + filter, + page, + perPage, + sortField, + sortOrder, + type: assistantAnonimizationFieldsTypeName, + fields, + }); + + return transformSavedObjectsToFoundAssistantAnonimizationField({ savedObjectsFindResponse }); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts index fb1c201196974..c61b726cea8a3 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts @@ -6,19 +6,29 @@ */ import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; -import type { SavedObjectsType } from '@kbn/core/server'; +import type { + SavedObject, + SavedObjectsFindResponse, + SavedObjectsType, + SavedObjectsUpdateResponse, +} from '@kbn/core/server'; +import { AnonimizationFieldResponse } from '../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; +import { FindAnonimizationFieldsResponse } from '../schemas/anonimization_fields/find_prompts_route.gen'; export const assistantAnonimizationFieldsTypeName = 'elastic-ai-assistant-anonimization-fields'; export const assistantAnonimizationFieldsTypeMappings: SavedObjectsType['mappings'] = { properties: { - fieldId: { + id: { type: 'keyword', }, - defaultAllow: { + field_id: { + type: 'keyword', + }, + default_allow: { type: 'boolean', }, - defaultAllowReplacement: { + default_allow_replacement: { type: 'boolean', }, updated_at: { @@ -36,10 +46,100 @@ export const assistantAnonimizationFieldsTypeMappings: SavedObjectsType['mapping }, }; +export const transformSavedObjectToAssistantAnonimizationField = ({ + savedObject, +}: { + savedObject: SavedObject; +}): AnonimizationFieldResponse => { + const { + version: _version, + attributes: { + /* eslint-disable @typescript-eslint/naming-convention */ + created_at, + created_by, + field_id, + default_allow, + default_allow_replacement, + updated_by, + updated_at, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + id, + } = savedObject; + + return { + createdAt: created_at, + createdBy: created_by, + fieldId: field_id, + defaultAllow: default_allow, + defaultAllowReplacement: default_allow_replacement, + updatedAt: updated_at, + updatedBy: updated_by, + id, + }; +}; + +export interface AssistantAnonimizationFieldSoSchema { + created_at: string; + created_by: string; + field_id: string; + default_allow?: boolean; + default_allow_replacement?: boolean; + updated_at: string; + updated_by: string; +} + export const assistantAnonimizationFieldsType: SavedObjectsType = { name: assistantAnonimizationFieldsTypeName, - indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, + indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, // todo: generic hidden: false, namespaceType: 'multiple-isolated', mappings: assistantAnonimizationFieldsTypeMappings, }; + +export const transformSavedObjectUpdateToAssistantAnonimizationField = ({ + savedObject, +}: { + savedObject: SavedObjectsUpdateResponse; +}): AnonimizationFieldResponse => { + const dateNow = new Date().toISOString(); + const { + version: _version, + attributes: { + updated_by: updatedBy, + default_allow: defaultAllow, + default_allow_replacement: defaultAllowReplacement, + created_at: createdAt, + created_by: createdBy, + field_id: fieldId, + }, + id, + updated_at: updatedAt, + } = savedObject; + + return { + createdAt, + createdBy, + fieldId: fieldId ?? '', + id, + defaultAllow, + defaultAllowReplacement, + updatedAt: updatedAt ?? dateNow, + updatedBy, + }; +}; + +export const transformSavedObjectsToFoundAssistantAnonimizationField = ({ + savedObjectsFindResponse, +}: { + savedObjectsFindResponse: SavedObjectsFindResponse; +}): FindAnonimizationFieldsResponse => { + return { + data: savedObjectsFindResponse.saved_objects.map((savedObject) => + transformSavedObjectToAssistantAnonimizationField({ savedObject }) + ), + page: savedObjectsFindResponse.page, + perPage: savedObjectsFindResponse.per_page, + total: savedObjectsFindResponse.total, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts new file mode 100644 index 0000000000000..9cc4c8e8f060f --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +export type BulkActionSkipReason = z.infer; +export const BulkActionSkipReason = z.literal('ANONIMIZATION_FIELD_NOT_MODIFIED'); + +export type BulkActionSkipResult = z.infer; +export const BulkActionSkipResult = z.object({ + id: z.string(), + name: z.string().optional(), + skip_reason: BulkActionSkipReason, +}); + +export type AnonimizationFieldDetailsInError = z.infer; +export const AnonimizationFieldDetailsInError = z.object({ + id: z.string(), + name: z.string().optional(), +}); + +export type NormalizedAnonimizationFieldError = z.infer; +export const NormalizedAnonimizationFieldError = z.object({ + message: z.string(), + status_code: z.number().int(), + err_code: z.string().optional(), + anonimization_fields: z.array(AnonimizationFieldDetailsInError), +}); + +export type AnonimizationFieldResponse = z.infer; +export const AnonimizationFieldResponse = z.object({ + id: z.string(), + fieldId: z.string(), + defaultAllow: z.boolean().optional(), + defaultAllowReplacement: z.boolean().optional(), + updatedAt: z.string().optional(), + updatedBy: z.string().optional(), + createdAt: z.string().optional(), + createdBy: z.string().optional(), +}); + +export type BulkCrudActionResults = z.infer; +export const BulkCrudActionResults = z.object({ + updated: z.array(AnonimizationFieldResponse), + created: z.array(AnonimizationFieldResponse), + deleted: z.array(z.string()), + skipped: z.array(BulkActionSkipResult), +}); + +export type BulkCrudActionSummary = z.infer; +export const BulkCrudActionSummary = z.object({ + failed: z.number().int(), + skipped: z.number().int(), + succeeded: z.number().int(), + total: z.number().int(), +}); + +export type BulkCrudActionResponse = z.infer; +export const BulkCrudActionResponse = z.object({ + success: z.boolean().optional(), + status_code: z.number().int().optional(), + message: z.string().optional(), + anonimization_fields_count: z.number().int().optional(), + attributes: z.object({ + results: BulkCrudActionResults, + summary: BulkCrudActionSummary, + errors: z.array(NormalizedAnonimizationFieldError).optional(), + }), +}); + +export type BulkActionBase = z.infer; +export const BulkActionBase = z.object({ + /** + * Query to filter anonimization fields + */ + query: z.string().optional(), + /** + * Array of anonimization fields IDs + */ + ids: z.array(z.string()).min(1).optional(), +}); + +export type AnonimizationFieldCreateProps = z.infer; +export const AnonimizationFieldCreateProps = z.object({ + fieldId: z.string(), + defaultAllow: z.boolean().optional(), + defaultAllowReplacement: z.boolean().optional(), +}); + +export type AnonimizationFieldUpdateProps = z.infer; +export const AnonimizationFieldUpdateProps = z.object({ + id: z.string(), + defaultAllow: z.boolean().optional(), + defaultAllowReplacement: z.boolean().optional(), +}); + +export type PerformBulkActionRequestBody = z.infer; +export const PerformBulkActionRequestBody = z.object({ + delete: BulkActionBase.optional(), + create: z.array(AnonimizationFieldCreateProps).optional(), + update: z.array(AnonimizationFieldUpdateProps).optional(), +}); +export type PerformBulkActionRequestBodyInput = z.input; + +export type PerformBulkActionResponse = z.infer; +export const PerformBulkActionResponse = BulkCrudActionResponse; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml new file mode 100644 index 0000000000000..204b0d8d93f20 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml @@ -0,0 +1,230 @@ +openapi: 3.0.0 +info: + title: Bulk Actions API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/anonimization_fields/_bulk_action: + post: + operationId: PerformBulkAction + x-codegen-enabled: true + summary: Applies a bulk action to multiple anonimization fields + description: The bulk action is applied to all anonimization fields that match the filter or to the list of anonimization fields by their IDs. + tags: + - Bulk API + requestBody: + content: + application/json: + schema: + type: object + properties: + delete: + $ref: '#/components/schemas/BulkActionBase' + create: + type: array + items: + $ref: '#/components/schemas/AnonimizationFieldCreateProps' + update: + type: array + items: + $ref: '#/components/schemas/AnonimizationFieldUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/BulkCrudActionResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + BulkActionSkipReason: + type: string + enum: + - ANONIMIZATION_FIELD_NOT_MODIFIED + + BulkActionSkipResult: + type: object + properties: + id: + type: string + name: + type: string + skip_reason: + $ref: '#/components/schemas/BulkActionSkipReason' + required: + - id + - skip_reason + + AnonimizationFieldDetailsInError: + type: object + properties: + id: + type: string + name: + type: string + required: + - id + + NormalizedAnonimizationFieldError: + type: object + properties: + message: + type: string + status_code: + type: integer + err_code: + type: string + anonimization_fields: + type: array + items: + $ref: '#/components/schemas/AnonimizationFieldDetailsInError' + required: + - message + - status_code + - anonimization_fields + + AnonimizationFieldResponse: + type: object + required: + - id + - fieldId + properties: + id: + type: string + fieldId: + type: string + defaultAllow: + type: boolean + defaultAllowReplacement: + type: boolean + updatedAt: + type: string + updatedBy: + type: string + createdAt: + type: string + createdBy: + type: string + + BulkCrudActionResults: + type: object + properties: + updated: + type: array + items: + $ref: '#/components/schemas/AnonimizationFieldResponse' + created: + type: array + items: + $ref: '#/components/schemas/AnonimizationFieldResponse' + deleted: + type: array + items: + type: string + skipped: + type: array + items: + $ref: '#/components/schemas/BulkActionSkipResult' + required: + - updated + - created + - deleted + - skipped + + BulkCrudActionSummary: + type: object + properties: + failed: + type: integer + skipped: + type: integer + succeeded: + type: integer + total: + type: integer + required: + - failed + - skipped + - succeeded + - total + + BulkCrudActionResponse: + type: object + properties: + success: + type: boolean + status_code: + type: integer + message: + type: string + anonimization_fields_count: + type: integer + attributes: + type: object + properties: + results: + $ref: '#/components/schemas/BulkCrudActionResults' + summary: + $ref: '#/components/schemas/BulkCrudActionSummary' + errors: + type: array + items: + $ref: '#/components/schemas/NormalizedAnonimizationFieldError' + required: + - results + - summary + required: + - attributes + + + BulkActionBase: + x-inline: true + type: object + properties: + query: + type: string + description: Query to filter anonimization fields + ids: + type: array + description: Array of anonimization fields IDs + minItems: 1 + items: + type: string + + AnonimizationFieldCreateProps: + type: object + required: + - fieldId + properties: + fieldId: + type: string + defaultAllow: + type: boolean + defaultAllowReplacement: + type: boolean + + AnonimizationFieldUpdateProps: + type: object + required: + - id + properties: + id: + type: string + defaultAllow: + type: boolean + defaultAllowReplacement: + type: boolean + \ No newline at end of file diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts new file mode 100644 index 0000000000000..7b6f638be5d5e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { ArrayFromString } from '@kbn/zod-helpers'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + */ + +import { AnonimizationFieldResponse } from './bulk_crud_anonimization_fields_route.gen'; + +export type FindAnonimizationFieldsSortField = z.infer; +export const FindAnonimizationFieldsSortField = z.enum([ + 'created_at', + 'is_default', + 'title', + 'updated_at', +]); +export type FindAnonimizationFieldsSortFieldEnum = typeof FindAnonimizationFieldsSortField.enum; +export const FindAnonimizationFieldsSortFieldEnum = FindAnonimizationFieldsSortField.enum; + +export type SortOrder = z.infer; +export const SortOrder = z.enum(['asc', 'desc']); +export type SortOrderEnum = typeof SortOrder.enum; +export const SortOrderEnum = SortOrder.enum; + +export type FindAnonimizationFieldsRequestQuery = z.infer< + typeof FindAnonimizationFieldsRequestQuery +>; +export const FindAnonimizationFieldsRequestQuery = z.object({ + fields: ArrayFromString(z.string()).optional(), + /** + * Search query + */ + filter: z.string().optional(), + /** + * Field to sort by + */ + sort_field: FindAnonimizationFieldsSortField.optional(), + /** + * Sort order + */ + sort_order: SortOrder.optional(), + /** + * Page number + */ + page: z.coerce.number().int().min(1).optional().default(1), + /** + * AnonimizationFields per page + */ + per_page: z.coerce.number().int().min(0).optional().default(20), +}); +export type FindAnonimizationFieldsRequestQueryInput = z.input< + typeof FindAnonimizationFieldsRequestQuery +>; + +export type FindAnonimizationFieldsResponse = z.infer; +export const FindAnonimizationFieldsResponse = z.object({ + page: z.number().int(), + perPage: z.number().int(), + total: z.number().int(), + data: z.array(AnonimizationFieldResponse), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml new file mode 100644 index 0000000000000..5c0e7c951363d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml @@ -0,0 +1,108 @@ +openapi: 3.0.0 +info: + title: Find AnonimizationFields API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/anonimization_fields/_find: + get: + operationId: FindAnonimizationFields + x-codegen-enabled: true + description: Finds anonimization fields that match the given query. + summary: Finds anonimization fields that match the given query. + tags: + - AnonimizationFields API + parameters: + - name: 'fields' + in: query + required: false + schema: + type: array + items: + type: string + - name: 'filter' + in: query + description: Search query + required: false + schema: + type: string + - name: 'sort_field' + in: query + description: Field to sort by + required: false + schema: + $ref: '#/components/schemas/FindAnonimizationFieldsSortField' + - name: 'sort_order' + in: query + description: Sort order + required: false + schema: + $ref: '#/components/schemas/SortOrder' + - name: 'page' + in: query + description: Page number + required: false + schema: + type: integer + minimum: 1 + default: 1 + - name: 'per_page' + in: query + description: AnonimizationFields per page + required: false + schema: + type: integer + minimum: 0 + default: 20 + + responses: + 200: + description: Successful response + content: + application/json: + schema: + type: object + properties: + page: + type: integer + perPage: + type: integer + total: + type: integer + data: + type: array + items: + $ref: './bulk_crud_anonimization_fields_route.schema.yaml#/components/schemas/AnonimizationFieldResponse' + required: + - page + - perPage + - total + - data + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + FindAnonimizationFieldsSortField: + type: string + enum: + - 'created_at' + - 'is_default' + - 'title' + - 'updated_at' + + SortOrder: + type: string + enum: + - 'asc' + - 'desc' diff --git a/x-pack/plugins/elastic_assistant/server/schemas/common.ts b/x-pack/plugins/elastic_assistant/server/schemas/common.ts deleted file mode 100644 index 00e97a9326c5e..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/common.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { fold } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; -import type * as rt from 'io-ts'; -import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; -import type { - RouteValidationFunction, - RouteValidationResultFactory, - RouteValidationError, -} from '@kbn/core/server'; - -type RequestValidationResult = - | { - value: T; - error?: undefined; - } - | { - value?: undefined; - error: RouteValidationError; - }; - -export const buildRouteValidation = - >(schema: T): RouteValidationFunction => - (inputValue: unknown, validationResult: RouteValidationResultFactory) => - pipe( - schema.decode(inputValue), - (decoded) => exactCheck(inputValue, decoded), - fold>( - (errors: rt.Errors) => validationResult.badRequest(formatErrors(errors).join()), - (validatedInput: A) => validationResult.ok(validatedInput) - ) - ); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts index b3dd9c2211cc1..1128c0e88a215 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts @@ -12,58 +12,15 @@ import { z } from 'zod'; * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. */ -/** - * AI assistant create KB settings. - */ -export type KnowledgeBaseCreateProps = z.infer; -export const KnowledgeBaseCreateProps = z.object({ - /** - * Prompt content. - */ - content: z.string(), - /** - * Prompt type. - */ - promptType: z.string(), - /** - * Is default prompt. - */ - isDefault: z.boolean().optional(), - /** - * Is shared prompt. - */ - isShared: z.boolean().optional(), - /** - * Is default prompt. - */ - isNewConversationDefault: z.boolean().optional(), -}); - /** * AI assistant KnowledgeBase. */ export type KnowledgeBaseResponse = z.infer; export const KnowledgeBaseResponse = z.object({ /** - * Prompt content. - */ - content: z.string(), - /** - * Prompt type. - */ - promptType: z.string().optional(), - /** - * Is default prompt. + * Identify the success of the method execution. */ - isDefault: z.boolean().optional(), - /** - * Is shared prompt. - */ - isShared: z.boolean().optional(), - /** - * Is default prompt. - */ - isNewConversationDefault: z.boolean().optional(), + success: z.boolean().optional(), }); export type CreateKnowledgeBaseRequestParams = z.infer; @@ -77,10 +34,6 @@ export type CreateKnowledgeBaseRequestParamsInput = z.input< typeof CreateKnowledgeBaseRequestParams >; -export type CreateKnowledgeBaseRequestBody = z.infer; -export const CreateKnowledgeBaseRequestBody = KnowledgeBaseCreateProps; -export type CreateKnowledgeBaseRequestBodyInput = z.input; - export type CreateKnowledgeBaseResponse = z.infer; export const CreateKnowledgeBaseResponse = KnowledgeBaseResponse; @@ -108,4 +61,8 @@ export const ReadKnowledgeBaseRequestParams = z.object({ export type ReadKnowledgeBaseRequestParamsInput = z.input; export type ReadKnowledgeBaseResponse = z.infer; -export const ReadKnowledgeBaseResponse = KnowledgeBaseResponse; +export const ReadKnowledgeBaseResponse = z.object({ + elser_exists: z.boolean().optional(), + index_exists: z.boolean().optional(), + pipeline_exists: z.boolean().optional(), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml index b3ab6e66af62d..fd0458bf29319 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: Create Conversation API endpoint + title: KnowledgeBase API endpoints version: '2023-10-31' paths: /internal/elastic_assistant/knowledge_base/{resource}: @@ -18,12 +18,6 @@ paths: description: The KnowledgeBase `resource` value. schema: type: string - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/KnowledgeBaseCreateProps' responses: 200: description: Indicates a successful call. @@ -64,7 +58,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/KnowledgeBaseResponse' + type: object + properties: + elser_exists: + type: boolean + index_exists: + type: boolean + pipeline_exists: + type: boolean 400: description: Generic Error content: @@ -81,7 +82,7 @@ paths: delete: operationId: DeleteKnowledgeBase x-codegen-enabled: true - description: Deletes KnowledgeBase using the `resource` field. + description: Deletes KnowledgeBase with the `resource` field. summary: Deletes a KnowledgeBase tags: - KnowledgeBase API @@ -115,49 +116,10 @@ paths: components: schemas: - KnowledgeBaseCreateProps: - type: object - description: AI assistant create KB settings. - required: - - 'content' - - 'promptType' - properties: - content: - type: string - description: Prompt content. - promptType: - type: string - description: Prompt type. - isDefault: - description: Is default prompt. - type: boolean - isShared: - description: Is shared prompt. - type: boolean - isNewConversationDefault: - description: Is default prompt. - type: boolean - KnowledgeBaseResponse: type: object description: AI assistant KnowledgeBase. - required: - - 'timestamp' - - 'content' - - 'role' properties: - content: - type: string - description: Prompt content. - promptType: - type: string - description: Prompt type. - isDefault: - description: Is default prompt. - type: boolean - isShared: - description: Is shared prompt. + success: type: boolean - isNewConversationDefault: - description: Is default prompt. - type: boolean + description: Identify the success of the method execution. diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/delete_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/delete_knowledge_base.ts deleted file mode 100644 index b891e18ed8f50..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/delete_knowledge_base.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -/** Validates the URL path of a DELETE request to the `/knowledge_base/{resource}` endpoint */ -export const DeleteKnowledgeBasePathParams = t.type({ - resource: t.union([t.string, t.undefined]), -}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/get_knowledge_base_status.ts b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/get_knowledge_base_status.ts deleted file mode 100644 index aa089fa86dce3..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/get_knowledge_base_status.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; - -/** Validates the URL path of a GET request to the `/knowledge_base/{resource}` endpoint */ -export const GetKnowledgeBaseStatusPathParams = t.type({ - resource: t.union([t.string, t.undefined]), -}); diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index 359441b02ea63..d82e89af95255 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -31,6 +31,7 @@ import { AIAssistantConversationsDataClient } from './conversations_data_client' import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; import { RequestBody } from './lib/langchain/types'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; +import { AIAssistantAnonimizationFieldsSOClient } from './saved_object/ai_assistant_anonimization_fields_so_client'; export const PLUGIN_ID = 'elasticAssistant' as const; @@ -100,6 +101,7 @@ export interface ElasticAssistantApiRequestHandlerContext { getCurrentUser: () => AuthenticatedUser | null; getAIAssistantConversationsDataClient: () => Promise; getAIAssistantPromptsSOClient: () => AIAssistantPromptsSOClient; + getAIAssistantAnonimizationFieldsSOClient: () => AIAssistantAnonimizationFieldsSOClient; telemetry: AnalyticsServiceSetup; } /** From dfdb03e63149a6ba61493590decbf7330a07a2a5 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 11:23:30 -0800 Subject: [PATCH 009/141] removed debug --- .../actions/server/sub_action_framework/sub_action_connector.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts b/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts index 01a1984187d43..7d3c6e51e844e 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/sub_action_connector.ts @@ -147,7 +147,6 @@ export abstract class SubActionConnector { return res; } catch (error) { - console.log('hdfjskhfjksdhfjskdhfsjkdfhsjkdfhsjkdfhsjkdfhsjkdfhjkdshfdksjfhsdjfhsdxjk') if (isAxiosError(error)) { this.logger.debug( `Request to external service failed. Connector Id: ${this.connector.id}. Connector type: ${this.connector.type}. Method: ${error.config?.method}. URL: ${error.config?.url}` From 71cd051341922ac3c5d3b13429d008c36e8e5f79 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 11:29:57 -0800 Subject: [PATCH 010/141] throw error if templated weren't installed --- .../elastic_assistant/server/ai_assistant_service/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index ef5f0b49e3a8d..da41a99ffe556 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -101,12 +101,15 @@ export class AIAssistantService { this.options.logger.debug(`Initializing resources for AIAssistantService`); const esClient = await this.options.elasticsearchClientPromise; - await this.conversationsDataStream.install({ + const installationResult = await this.conversationsDataStream.install({ esClient, logger: this.options.logger, pluginStop$: this.options.pluginStop$, }); + if (installationResult.error !== undefined) { + throw installationResult.error; + } this.initialized = true; this.isInitializing = false; return successResult(); From 6ad0564cbec753bc04291ca9c77d4c502814319f Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 12:14:27 -0800 Subject: [PATCH 011/141] replaced context method getConversation with storage API request to get lastConversationId from the server --- .../impl/assistant/assistant_overlay/index.tsx | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx index cb075e26f4df4..62100d29015d0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx @@ -14,7 +14,7 @@ import styled from 'styled-components'; import { ShowAssistantOverlayProps, useAssistantContext } from '../../assistant_context'; import { Assistant } from '..'; import { WELCOME_CONVERSATION_TITLE } from '../use_conversation/translations'; -import { useFetchCurrentUserConversations } from '../api/conversations/use_fetch_current_user_conversations'; +import { useLastConversation } from '../api/conversations/use_fetch_current_user_conversations'; const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; @@ -36,19 +36,14 @@ export const AssistantOverlay = React.memo(() => { const [promptContextId, setPromptContextId] = useState(); const { assistantTelemetry, setShowAssistantOverlay } = useAssistantContext(); - const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); + const { data: lastConversation, isLoading } = useLastConversation(); const lastConversationId = useMemo(() => { if (!isLoading) { - const sorted = conversationsData?.data.sort((convA, convB) => - convA.updatedAt && convB.updatedAt && convA.updatedAt > convB.updatedAt ? -1 : 1 - ); - if (sorted && sorted.length > 0) { - return sorted[0].id; - } + return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; } return WELCOME_CONVERSATION_TITLE; - }, [conversationsData?.data, isLoading]); + }, [isLoading, lastConversation?.id]); // Bind `showAssistantOverlay` in SecurityAssistantContext to this modal instance const showOverlay = useCallback( From c4a83af9c773ba778ff32a1793d6d23345fa639e Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 12:42:05 -0800 Subject: [PATCH 012/141] added refetch current conversation on useChatSent --- .../transform_raw_data/index.test.tsx | 6 +++--- .../transform_raw_data/index.tsx | 10 ++++++---- .../impl/assistant/api/index.tsx | 6 ++++-- .../impl/assistant/chat_send/use_chat_send.tsx | 6 +++--- .../impl/assistant/index.tsx | 12 ++++++++++-- .../impl/assistant/prompt/helpers.ts | 8 +++++--- .../impl/assistant/use_send_messages/index.tsx | 4 +++- .../impl/assistant_context/constants.tsx | 1 - .../impl/assistant_context/index.tsx | 15 --------------- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx index fb8b1c5b42f6b..ded518deece66 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx @@ -22,7 +22,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: () => {}, + onNewReplacements: jest.fn(), rawData: inputRawData.rawData, }); @@ -64,7 +64,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: () => {}, + onNewReplacements: jest.fn(), rawData: inputRawData.rawData, }); @@ -88,7 +88,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: () => {}, + onNewReplacements: jest.fn(), rawData: inputRawData.rawData, }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx index f1fe5e9331344..5c621304d7c8e 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx @@ -9,7 +9,7 @@ import { getAnonymizedData } from '../get_anonymized_data'; import { getAnonymizedValues } from '../get_anonymized_values'; import { getCsvFromData } from '../get_csv_from_data'; -export const transformRawData = ({ +export const transformRawData = async ({ allow, allowReplacement, currentReplacements, @@ -27,9 +27,11 @@ export const transformRawData = ({ currentReplacements: Record | undefined; rawValue: string; }) => string; - onNewReplacements?: (replacements: Record) => void; + onNewReplacements: ( + replacements: Record + ) => Promise | undefined>; rawData: string | Record; -}): string => { +}): Promise => { if (typeof rawData === 'string') { return rawData; } @@ -44,7 +46,7 @@ export const transformRawData = ({ }); if (onNewReplacements != null) { - onNewReplacements(anonymizedData.replacements); + await onNewReplacements(anonymizedData.replacements); } return getCsvFromData(anonymizedData.anonymizedData); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index 55d095ca54c74..3d2ec06cef544 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -30,7 +30,9 @@ export interface FetchConnectorExecuteAction { apiConfig: Conversation['apiConfig']; http: HttpSetup; messages: Message[]; - onNewReplacements: (newReplacements: Record) => void; + onNewReplacements: ( + newReplacements: Record + ) => Promise | undefined>; replacements?: Record; signal?: AbortSignal | undefined; size?: number; @@ -189,7 +191,7 @@ export const fetchConnectorExecuteAction = async ({ } : undefined; - onNewReplacements(response.replacements ?? {}); + await onNewReplacements(response.replacements ?? {}); return { response: hasParsableResponse({ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 16ddf0447aeb4..8ed592a77c79a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -27,7 +27,7 @@ export interface UseChatSendProps { React.SetStateAction> >; setUserPrompt: React.Dispatch>; - refresh: () => Promise | undefined>; + refresh: () => Promise; } export interface UseChatSend { @@ -68,7 +68,7 @@ export const useChatSend = ({ // Handles sending latest user prompt to API const handleSendMessage = useCallback( async (promptText: string) => { - const onNewReplacements = (newReplacements: Record) => + const onNewReplacements = async (newReplacements: Record) => appendReplacements({ conversationId: currentConversation.id, replacements: newReplacements, @@ -125,7 +125,7 @@ export const useChatSend = ({ ); const handleRegenerateResponse = useCallback(async () => { - const onNewReplacements = (newReplacements: Record) => + const onNewReplacements = async (newReplacements: Record) => appendReplacements({ conversationId: currentConversation.id, replacements: newReplacements, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index ce98ace90fa5e..ab8f23f687e28 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -101,7 +101,7 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); - const { amendMessage, getDefaultConversation } = useConversation(); + const { amendMessage, getDefaultConversation, getConversation } = useConversation(); const { data: conversationsData, isLoading, refetch } = useFetchCurrentUserConversations(); const { data: lastConversation, isLoading: isLoadingLast } = useLastConversation(); @@ -189,6 +189,14 @@ const AssistantComponent: React.FC = ({ getDefaultConversation({ conversationId: selectedConversationId }) ); + const refetchCurrentConversation = useCallback(async () => { + const updatedConversation = await getConversation(selectedConversationId); + if (updatedConversation) { + setCurrentConversation(updatedConversation); + } + return updatedConversation; + }, [getConversation, selectedConversationId]); + useEffect(() => { if (!isLoadingLast && lastConversation && lastConversation.id) { setCurrentConversation(lastConversation); @@ -422,7 +430,7 @@ const AssistantComponent: React.FC = ({ setEditingSystemPromptId, selectedPromptContexts, setSelectedPromptContexts, - refresh: refetchResults, + refresh: refetchCurrentConversation, }); const chatbotComments = useMemo( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index 26058e05ee697..fa0cd0221bf0a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -51,15 +51,17 @@ export async function getCombinedMessage({ rawValue: string; }) => string; isNewChat: boolean; - onNewReplacements: (newReplacements: Record) => void; + onNewReplacements: ( + newReplacements: Record + ) => Promise | undefined>; promptText: string; selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; }): Promise { const promptContextsContent = Object.keys(selectedPromptContexts) .sort() - .map((id) => { - const promptContext = transformRawData({ + .map(async (id) => { + const promptContext = await transformRawData({ allow: selectedPromptContexts[id].allow, allowReplacement: selectedPromptContexts[id].allowReplacement, currentReplacements, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx index eae7d7914e6a1..af28a653b59e2 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx @@ -18,7 +18,9 @@ interface SendMessagesProps { apiConfig: Conversation['apiConfig']; http: HttpSetup; messages: Message[]; - onNewReplacements: (newReplacements: Record) => void; + onNewReplacements: ( + newReplacements: Record + ) => Promise | undefined>; replacements?: Record; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx index cc747a705b851..b435e11256359 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx @@ -10,7 +10,6 @@ import { KnowledgeBaseConfig } from '../assistant/types'; export const DEFAULT_ASSISTANT_NAMESPACE = 'elasticAssistantDefault'; export const QUICK_PROMPT_LOCAL_STORAGE_KEY = 'quickPrompts'; export const SYSTEM_PROMPT_LOCAL_STORAGE_KEY = 'systemPrompts'; -export const LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY = 'lastConversationId'; export const KNOWLEDGE_BASE_LOCAL_STORAGE_KEY = 'knowledgeBase'; /** The default `n` latest alerts, ordered by risk score, sent as context to the assistant */ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 26686afa77607..30addae96583e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -14,7 +14,6 @@ import { ActionTypeRegistryContract } from '@kbn/triggers-actions-ui-plugin/publ import { useLocalStorage } from 'react-use'; import type { DocLinksStart } from '@kbn/core-doc-links-browser'; import { defaultAssistantFeatures } from '@kbn/elastic-assistant-common'; -import { WELCOME_CONVERSATION_TITLE } from '../assistant/use_conversation/translations'; import { updatePromptContexts } from './helpers'; import type { PromptContext, @@ -32,7 +31,6 @@ import { DEFAULT_ASSISTANT_NAMESPACE, DEFAULT_KNOWLEDGE_BASE_SETTINGS, KNOWLEDGE_BASE_LOCAL_STORAGE_KEY, - LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY, QUICK_PROMPT_LOCAL_STORAGE_KEY, SYSTEM_PROMPT_LOCAL_STORAGE_KEY, } from './constants'; @@ -197,9 +195,6 @@ export const AssistantProvider: React.FC = ({ baseSystemPrompts ); - const [localStorageLastConversationId, setLocalStorageLastConversationId] = - useLocalStorage(`${nameSpace}.${LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY}`); - /** * Local storage for knowledge base configuration, prefixed by assistant nameSpace */ @@ -253,14 +248,6 @@ export const AssistantProvider: React.FC = ({ */ const [selectedSettingsTab, setSelectedSettingsTab] = useState(CONVERSATIONS_TAB); - const getConversationId = useCallback( - // if a conversationId has been provided, use that - // if not, check local storage - // last resort, go to welcome conversation - (id?: string) => id ?? localStorageLastConversationId ?? WELCOME_CONVERSATION_TITLE, - [localStorageLastConversationId] - ); - // Fetch assistant capabilities const { data: capabilities } = useCapabilities({ http, toasts }); const { assistantModelEvaluation: modelEvaluatorEnabled, assistantStreamingEnabled } = @@ -305,7 +292,6 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, - setLastConversationId: setLocalStorageLastConversationId, baseConversations, }), [ @@ -337,7 +323,6 @@ export const AssistantProvider: React.FC = ({ setDefaultAllow, setDefaultAllowReplacement, setLocalStorageKnowledgeBase, - setLocalStorageLastConversationId, setLocalStorageQuickPrompts, setLocalStorageSystemPrompts, showAssistantOverlay, From 418a51c8f0391e3f07d24b71832160d25542e308 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 14 Jan 2024 13:59:34 -0800 Subject: [PATCH 013/141] cleaned up select handler --- .../impl/assistant/assistant_header/index.tsx | 2 -- .../conversations/conversation_selector/index.tsx | 4 ++-- .../kbn-elastic-assistant/impl/assistant/index.tsx | 11 +++++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index da454944692e7..4c7139adfc9a0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -33,7 +33,6 @@ interface OwnProps { isSettingsModalVisible: boolean; onConversationSelected: (cId: string) => void; onToggleShowAnonymizedValues: (e: EuiSwitchEvent) => void; - selectedConversationId: string; setIsSettingsModalVisible: React.Dispatch>; setSelectedConversationId: React.Dispatch>; setCurrentConversation: React.Dispatch>; @@ -57,7 +56,6 @@ export const AssistantHeader: React.FC = ({ isSettingsModalVisible, onConversationSelected, onToggleShowAnonymizedValues, - selectedConversationId, setIsSettingsModalVisible, setSelectedConversationId, shouldDisableKeyboardShortcut, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 2c5462d0e07ae..5677279829e71 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -34,7 +34,7 @@ interface Props { defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; selectedConversationId: string | undefined; - onConversationSelected: (conversationTitle: string, conversationId?: string) => void; + onConversationSelected: (conversationId: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; } @@ -140,7 +140,7 @@ export const ConversationSelector: React.FC = React.memo( }; cId = (await createConversation(newConversation))?.id; } - onConversationSelected(searchValue, cId); + onConversationSelected(cId ?? DEFAULT_CONVERSATION_TITLE); }, [ allSystemPrompts, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index ab8f23f687e28..bbfc7814a9531 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -297,8 +297,8 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - async (cTitle: string, cId?: string) => { - if (conversations[cTitle] === undefined && cId) { + async (cId: string) => { + if (conversations[cId] === undefined && cId) { const updatedConv = await refetchResults(); if (updatedConv) { setCurrentConversation(updatedConv[cId]); @@ -307,9 +307,9 @@ const AssistantComponent: React.FC = ({ getDefaultSystemPrompt({ allSystemPrompts, conversation: updatedConv[cId] })?.id ); } - } else { - setSelectedConversationId(cTitle); - setCurrentConversation(conversations[cTitle]); + } else if (cId) { + setSelectedConversationId(cId); + setCurrentConversation(conversations[cId]); setEditingSystemPromptId( getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation })?.id ); @@ -530,7 +530,6 @@ const AssistantComponent: React.FC = ({ isSettingsModalVisible={isSettingsModalVisible} onConversationSelected={handleOnConversationSelected} onToggleShowAnonymizedValues={onToggleShowAnonymizedValues} - selectedConversationId={selectedConversationId} setIsSettingsModalVisible={setIsSettingsModalVisible} setSelectedConversationId={setSelectedConversationId} showAnonymizedValues={showAnonymizedValues} From 6ebff9beefc45d76726640fb334d6d5d03fceb7c Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 16 Jan 2024 23:27:22 -0800 Subject: [PATCH 014/141] fixed ui --- .../use_bulk_actions_conversations.ts | 22 ++- .../impl/assistant/assistant_header/index.tsx | 7 + .../conversation_selector/index.tsx | 58 ++----- .../conversation_selector_settings/index.tsx | 9 +- .../conversation_settings.tsx | 157 +++++++++++++----- .../impl/assistant/index.tsx | 47 ++++-- .../assistant/settings/assistant_settings.tsx | 17 +- .../settings/assistant_settings_button.tsx | 3 + .../use_settings_updater.tsx | 63 ++++--- .../conversations_data_writer.ts | 34 +++- .../create_conversation.ts | 3 +- .../server/conversations_data_client/index.ts | 3 +- .../routes/conversation/bulk_actions_route.ts | 7 +- 13 files changed, 277 insertions(+), 153 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index c1dcb78458e88..91cbf6e13f4f4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -7,10 +7,12 @@ // import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; import { HttpSetup } from '@kbn/core/public'; +import { + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; import { Conversation } from '../../../assistant_context/types'; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_KEY = 'elastic_assistant_conversations'; - export interface BulkActionSummary { failed: number; skipped: number; @@ -46,15 +48,17 @@ export interface BulkUpdateResponse { export const bulkConversationsChange = ( http: HttpSetup, - conversations: { - conversationsToUpdate?: Conversation[]; - conversationsToCreate?: Conversation[]; - conversationsToDelete?: string[]; + conversationsActions: { + update?: Array>; + create?: Conversation[]; + delete?: { + ids: string[]; + }; } ) => { - return http.fetch(`/api/elastic_assistant/conversations/_bulk_action`, { + return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { method: 'POST', - version: '2023-10-31', - body: JSON.stringify(conversations), + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + body: JSON.stringify(conversationsActions), }); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index 4c7139adfc9a0..2fd7e57c3fe70 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -32,6 +32,7 @@ interface OwnProps { isDisabled: boolean; isSettingsModalVisible: boolean; onConversationSelected: (cId: string) => void; + onConversationDeleted: (conversationId: string) => void; onToggleShowAnonymizedValues: (e: EuiSwitchEvent) => void; setIsSettingsModalVisible: React.Dispatch>; setSelectedConversationId: React.Dispatch>; @@ -39,6 +40,7 @@ interface OwnProps { shouldDisableKeyboardShortcut?: () => boolean; showAnonymizedValues: boolean; title: string | JSX.Element; + conversations: Record; } type Props = OwnProps; @@ -55,6 +57,7 @@ export const AssistantHeader: React.FC = ({ isDisabled, isSettingsModalVisible, onConversationSelected, + onConversationDeleted, onToggleShowAnonymizedValues, setIsSettingsModalVisible, setSelectedConversationId, @@ -62,6 +65,7 @@ export const AssistantHeader: React.FC = ({ showAnonymizedValues, title, setCurrentConversation, + conversations, }) => { const showAnonymizedValuesChecked = useMemo( () => @@ -102,6 +106,8 @@ export const AssistantHeader: React.FC = ({ onConversationSelected={onConversationSelected} shouldDisableKeyboardShortcut={shouldDisableKeyboardShortcut} isDisabled={isDisabled} + conversations={conversations} + onConversationDeleted={onConversationDeleted} /> <> @@ -133,6 +139,7 @@ export const AssistantHeader: React.FC = ({ selectedConversation={currentConversation} setIsSettingsModalVisible={setIsSettingsModalVisible} setSelectedConversationId={setSelectedConversationId} + conversations={conversations} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 5677279829e71..572fe9b0a2385 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -20,8 +20,7 @@ import useEvent from 'react-use/lib/useEvent'; import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; -import { merge } from 'lodash'; -import { Conversation, useFetchCurrentUserConversations } from '../../../..'; +import { Conversation } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import * as i18n from './translations'; import { DEFAULT_CONVERSATION_TITLE } from '../../use_conversation/translations'; @@ -35,8 +34,10 @@ interface Props { defaultProvider?: OpenAiProviderType; selectedConversationId: string | undefined; onConversationSelected: (conversationId: string) => void; + onConversationDeleted: (conversationId: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; + conversations: Record; } const getPreviousConversationId = (conversationIds: string[], selectedConversationId: string) => { @@ -61,40 +62,14 @@ export const ConversationSelector: React.FC = React.memo( defaultConnectorId, defaultProvider, onConversationSelected, + onConversationDeleted, shouldDisableKeyboardShortcut = () => false, isDisabled = false, + conversations, }) => { - const [conversations, setConversations] = useState>({}); - const { allSystemPrompts, baseConversations } = useAssistantContext(); + const { allSystemPrompts } = useAssistantContext(); - const { deleteConversation, createConversation } = useConversation(); - - const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); - - useEffect(() => { - if (!isLoading) { - const userConversations = (conversationsData?.data ?? []).reduce< - Record - >((transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, {}); - setConversations( - merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ) - ); - } - }, [baseConversations, conversationsData?.data, isLoading]); + const { createConversation } = useConversation(); const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); const conversationOptions = useMemo(() => { @@ -157,28 +132,17 @@ export const ConversationSelector: React.FC = React.memo( if (selectedConversationId === cId) { onConversationSelected(getPreviousConversationId(conversationIds, cId)); } - setTimeout(() => { - deleteConversation(cId); - }, 0); - const deletedConv = { ...conversations }; - delete deletedConv[cId]; - setConversations(deletedConv); + onConversationDeleted(cId); }, - [ - selectedConversationId, - conversations, - onConversationSelected, - conversationIds, - deleteConversation, - ] + [selectedConversationId, onConversationDeleted, onConversationSelected, conversationIds] ); const onChange = useCallback( - (newOptions: ConversationSelectorOption[]) => { + async (newOptions: ConversationSelectorOption[]) => { if (newOptions.length === 0 || !newOptions?.[0].id) { setSelectedOptions([]); } else if (conversationOptions.findIndex((o) => o.id === newOptions?.[0].id) !== -1) { - onConversationSelected(newOptions?.[0].id); + await onConversationSelected(newOptions?.[0].id); } }, [conversationOptions, onConversationSelected] diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index 77692bc80f394..1ba589102fcb4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -72,7 +72,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( return Object.values(conversations).map((conversation) => ({ value: { isDefault: conversation.isDefault ?? false }, label: conversation.title, - id: conversation.id ?? '', + id: conversation.id ?? conversation.title, 'data-test-subj': conversation.id, })); }); @@ -89,9 +89,9 @@ export const ConversationSelectorSettings: React.FC = React.memo( conversationSelectorSettingsOption.length === 0 ? undefined : Object.values(conversations).find( - (conversation) => - conversation.title === conversationSelectorSettingsOption[0]?.label - ) ?? conversationSelectorSettingsOption[0]?.label; + (conversation) => conversation.id === conversationSelectorSettingsOption[0]?.id + ) ?? conversationSelectorSettingsOption[0]?.id; + onConversationSelectionChange(newConversation); }, [onConversationSelectionChange, conversations] @@ -114,6 +114,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( const newOption = { value: searchValue, label: searchValue, + id: searchValue, }; if (!optionExists) { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index d3001f00de603..41311ce04d14f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -27,6 +27,8 @@ import { getGenAiConfig } from '../../../connectorland/helpers'; export interface ConversationSettingsProps { allSystemPrompts: Prompt[]; conversationSettings: Record; + createdConversationSettings: Record; + deletedConversationSettings: string[]; defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; http: HttpSetup; @@ -35,6 +37,10 @@ export interface ConversationSettingsProps { setUpdatedConversationSettings: React.Dispatch< React.SetStateAction> >; + setDeletedConversationSettings: React.Dispatch>; + setCreatedConversationSettings: React.Dispatch< + React.SetStateAction> + >; isDisabled?: boolean; } @@ -49,9 +55,13 @@ export const ConversationSettings: React.FC = React.m selectedConversation, onSelectedConversationChange, conversationSettings, + createdConversationSettings, + deletedConversationSettings, http, setUpdatedConversationSettings, isDisabled = false, + setCreatedConversationSettings, + setDeletedConversationSettings, }) => { const defaultSystemPrompt = useMemo(() => { return getDefaultSystemPrompt({ allSystemPrompts, conversation: undefined }); @@ -81,7 +91,15 @@ export const ConversationSettings: React.FC = React.m } : c; - if (newSelectedConversation != null) { + if ( + newSelectedConversation && + (isNew || newSelectedConversation.id === newSelectedConversation.title) + ) { + setCreatedConversationSettings({ + ...createdConversationSettings, + [isNew ? c : newSelectedConversation.id]: newSelectedConversation, + }); + } else if (newSelectedConversation != null) { setUpdatedConversationSettings((prev) => { return { ...prev, @@ -93,43 +111,58 @@ export const ConversationSettings: React.FC = React.m onSelectedConversationChange(newSelectedConversation); }, [ + createdConversationSettings, defaultConnectorId, defaultProvider, defaultSystemPrompt?.id, onSelectedConversationChange, + setCreatedConversationSettings, setUpdatedConversationSettings, ] ); const onConversationDeleted = useCallback( (conversationId: string) => { - setUpdatedConversationSettings((prev) => { - const { [conversationId]: prevConversation, ...updatedConversations } = prev; - if (prevConversation != null) { - return updatedConversations; - } - return prev; - }); + setDeletedConversationSettings([...deletedConversationSettings, conversationId]); }, - [setUpdatedConversationSettings] + [deletedConversationSettings, setDeletedConversationSettings] ); const handleOnSystemPromptSelectionChange = useCallback( (systemPromptId?: string | undefined) => { if (selectedConversation != null) { - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - defaultSystemPromptId: systemPromptId, + if (conversationSettings[selectedConversation.id]) { + setUpdatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + defaultSystemPromptId: systemPromptId, + }, + updatedAt: undefined, + }, + })); + } else { + setCreatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + defaultSystemPromptId: systemPromptId, + }, }, - }, - })); + })); + } } }, - [selectedConversation, setUpdatedConversationSettings] + [ + conversationSettings, + selectedConversation, + setCreatedConversationSettings, + setUpdatedConversationSettings, + ] ); const selectedConnector = useMemo(() => { @@ -150,21 +183,42 @@ export const ConversationSettings: React.FC = React.m if (selectedConversation != null) { const config = getGenAiConfig(connector); - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - connectorId: connector?.id, - provider: config?.apiProvider, - model: config?.defaultModel, + if (conversationSettings[selectedConversation.id]) { + setUpdatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + connectorId: connector?.id, + provider: config?.apiProvider, + model: config?.defaultModel, + }, + updatedAt: undefined, }, - }, - })); + })); + } else { + setCreatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + connectorId: connector?.id, + provider: config?.apiProvider, + model: config?.defaultModel, + }, + }, + })); + } } }, - [selectedConversation, setUpdatedConversationSettings] + [ + conversationSettings, + selectedConversation, + setCreatedConversationSettings, + setUpdatedConversationSettings, + ] ); const selectedModel = useMemo(() => { @@ -176,19 +230,38 @@ export const ConversationSettings: React.FC = React.m const handleOnModelSelectionChange = useCallback( (model?: string) => { if (selectedConversation != null) { - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - model, + if (conversationSettings[selectedConversation.id]) { + setUpdatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + model, + }, + updatedAt: undefined, + }, + })); + } else { + setCreatedConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + model, + }, }, - }, - })); + })); + } } }, - [selectedConversation, setUpdatedConversationSettings] + [ + conversationSettings, + selectedConversation, + setCreatedConversationSettings, + setUpdatedConversationSettings, + ] ); return ( @@ -202,7 +275,7 @@ export const ConversationSettings: React.FC = React.m diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index bbfc7814a9531..56004c55089d2 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -101,7 +101,8 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); - const { amendMessage, getDefaultConversation, getConversation } = useConversation(); + const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = + useConversation(); const { data: conversationsData, isLoading, refetch } = useFetchCurrentUserConversations(); const { data: lastConversation, isLoading: isLoadingLast } = useLastConversation(); @@ -189,13 +190,16 @@ const AssistantComponent: React.FC = ({ getDefaultConversation({ conversationId: selectedConversationId }) ); - const refetchCurrentConversation = useCallback(async () => { - const updatedConversation = await getConversation(selectedConversationId); - if (updatedConversation) { - setCurrentConversation(updatedConversation); - } - return updatedConversation; - }, [getConversation, selectedConversationId]); + const refetchCurrentConversation = useCallback( + async (cId?: string) => { + const updatedConversation = await getConversation(cId ?? selectedConversationId); + if (updatedConversation) { + setCurrentConversation(updatedConversation); + } + return updatedConversation; + }, + [getConversation, selectedConversationId] + ); useEffect(() => { if (!isLoadingLast && lastConversation && lastConversation.id) { @@ -309,13 +313,34 @@ const AssistantComponent: React.FC = ({ } } else if (cId) { setSelectedConversationId(cId); - setCurrentConversation(conversations[cId]); + const refetchedConversation = await refetchCurrentConversation(cId); + if (refetchedConversation) { + setCurrentConversation(refetchedConversation); + } setEditingSystemPromptId( getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation })?.id ); } }, - [allSystemPrompts, conversations, currentConversation, refetchResults] + [ + allSystemPrompts, + conversations, + currentConversation, + refetchCurrentConversation, + refetchResults, + ] + ); + + const handleOnConversationDeleted = useCallback( + async (cId: string) => { + setTimeout(() => { + deleteConversation(cId); + }, 0); + const deletedConv = { ...conversations }; + delete deletedConv[cId]; + setConversations(deletedConv); + }, + [conversations, deleteConversation] ); const handleOnSystemPromptSelectionChange = useCallback((systemPromptId?: string) => { @@ -534,6 +559,8 @@ const AssistantComponent: React.FC = ({ setSelectedConversationId={setSelectedConversationId} showAnonymizedValues={showAnonymizedValues} title={currentTitle} + conversations={conversations} + onConversationDeleted={handleOnConversationDeleted} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index 9c7a176c28ccd..e34eb983eb8b9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -66,6 +66,7 @@ interface Props { onSave: () => void; selectedConversation: Conversation; setSelectedConversationId: React.Dispatch>; + conversations: Record; } /** @@ -80,6 +81,7 @@ export const AssistantSettings: React.FC = React.memo( onSave, selectedConversation: defaultSelectedConversation, setSelectedConversationId, + conversations, }) => { const { modelEvaluatorEnabled, http, selectedSettingsTab, setSelectedSettingsTab } = useAssistantContext(); @@ -88,6 +90,7 @@ export const AssistantSettings: React.FC = React.memo( conversationSettings, defaultAllow, defaultAllowReplacement, + deletedConversationSettings, knowledgeBase, quickPromptSettings, systemPromptSettings, @@ -98,7 +101,10 @@ export const AssistantSettings: React.FC = React.memo( setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, saveSettings, - } = useSettingsUpdater(); + createdConversationSettings, + setCreatedConversationSettings, + setDeletedConversationSettings, + } = useSettingsUpdater(conversations); // Local state for saving previously selected items so tab switching is friendlier // Conversation Selection State @@ -112,9 +118,10 @@ export const AssistantSettings: React.FC = React.memo( }, []); useEffect(() => { if (selectedConversation != null) { - setSelectedConversation(conversationSettings[selectedConversation.id]); + const v = conversationSettings[selectedConversation.id]; + setSelectedConversation(v ? v : createdConversationSettings[selectedConversation.id]); } - }, [conversationSettings, selectedConversation]); + }, [conversationSettings, createdConversationSettings, selectedConversation]); // Quick Prompt Selection State const [selectedQuickPrompt, setSelectedQuickPrompt] = useState(); @@ -282,7 +289,11 @@ export const AssistantSettings: React.FC = React.memo( defaultConnectorId={defaultConnectorId} defaultProvider={defaultProvider} conversationSettings={conversationSettings} + createdConversationSettings={createdConversationSettings} + deletedConversationSettings={deletedConversationSettings} setUpdatedConversationSettings={setUpdatedConversationSettings} + setCreatedConversationSettings={setCreatedConversationSettings} + setDeletedConversationSettings={setDeletedConversationSettings} allSystemPrompts={systemPromptSettings} selectedConversation={selectedConversation} isDisabled={selectedConversation == null} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index fe7f2217f8c1d..40a3251d18cd6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -22,6 +22,7 @@ interface Props { setIsSettingsModalVisible: React.Dispatch>; setSelectedConversationId: React.Dispatch>; isDisabled?: boolean; + conversations: Record; } /** @@ -36,6 +37,7 @@ export const AssistantSettingsButton: React.FC = React.memo( setIsSettingsModalVisible, selectedConversation, setSelectedConversationId, + conversations, }) => { const { toasts, setSelectedSettingsTab } = useAssistantContext(); @@ -82,6 +84,7 @@ export const AssistantSettingsButton: React.FC = React.memo( setSelectedConversationId={setSelectedConversationId} onClose={handleCloseModal} onSave={handleSave} + conversations={conversations} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 4a2b03b91a090..81f8c78cbe518 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -6,14 +6,15 @@ */ import React, { useCallback, useState } from 'react'; -import { merge } from 'lodash'; -import { Conversation, Prompt, QuickPrompt, useFetchCurrentUserConversations } from '../../../..'; +import { Conversation, Prompt, QuickPrompt } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; import { bulkConversationsChange } from '../../api/conversations/use_bulk_actions_conversations'; interface UseSettingsUpdater { conversationSettings: Record; + createdConversationSettings: Record; + deletedConversationSettings: string[]; defaultAllow: string[]; defaultAllowReplacement: string[]; knowledgeBase: KnowledgeBaseConfig; @@ -25,6 +26,9 @@ interface UseSettingsUpdater { setUpdatedConversationSettings: React.Dispatch< React.SetStateAction> >; + setCreatedConversationSettings: React.Dispatch< + React.SetStateAction> + >; setUpdatedKnowledgeBaseSettings: React.Dispatch>; setUpdatedQuickPromptSettings: React.Dispatch>; setUpdatedSystemPromptSettings: React.Dispatch>; @@ -32,7 +36,9 @@ interface UseSettingsUpdater { saveSettings: () => void; } -export const useSettingsUpdater = (): UseSettingsUpdater => { +export const useSettingsUpdater = ( + conversations: Record +): UseSettingsUpdater => { // Initial state from assistant context const { allQuickPrompts, @@ -40,7 +46,6 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { assistantTelemetry, defaultAllow, defaultAllowReplacement, - baseConversations, knowledgeBase, setAllQuickPrompts, setAllSystemPrompts, @@ -50,25 +55,15 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { http, } = useAssistantContext(); - const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); - - const conversations = merge( - baseConversations, - (conversationsData?.data ?? []).reduce>( - (transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, - {} - ) - ); - /** * Pending updating state */ // Conversations const [updatedConversationSettings, setUpdatedConversationSettings] = useState>(conversations); + const [createdConversationSettings, setCreatedConversationSettings] = useState< + Record + >({}); const [deletedConversationSettings, setDeletedConversationSettings] = useState([]); // Quick Prompts const [updatedQuickPromptSettings, setUpdatedQuickPromptSettings] = @@ -90,6 +85,7 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { const resetSettings = useCallback((): void => { setUpdatedConversationSettings(conversations); setDeletedConversationSettings([]); + setCreatedConversationSettings({}); setUpdatedQuickPromptSettings(allQuickPrompts); setUpdatedKnowledgeBaseSettings(knowledgeBase); setUpdatedSystemPromptSettings(allSystemPrompts); @@ -111,25 +107,36 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); bulkConversationsChange(http, { - conversationsToUpdate: Object.keys(updatedConversationSettings).reduce( - (conversationsToUpdate: Conversation[], conversationId: string) => { - if (!updatedConversationSettings[conversationId].isDefault) { - conversationsToUpdate.push(updatedConversationSettings[conversationId]); + update: Object.keys(updatedConversationSettings).reduce( + ( + conversationsToUpdate: Array>, + conversationId: string + ) => { + if (updatedConversationSettings.updatedAt === undefined) { + conversationsToUpdate.push({ + ...(updatedConversationSettings[conversationId] as Omit< + Conversation, + 'createdAt' | 'updatedAt' | 'user' + >), + }); } return conversationsToUpdate; }, [] ), - conversationsToCreate: Object.keys(updatedConversationSettings).reduce( + create: Object.keys(createdConversationSettings).reduce( (conversationsToCreate: Conversation[], conversationId: string) => { - if (updatedConversationSettings[conversationId].isDefault) { - conversationsToCreate.push(updatedConversationSettings[conversationId]); - } + conversationsToCreate.push(createdConversationSettings[conversationId]); return conversationsToCreate; }, [] ), - conversationsToDelete: deletedConversationSettings, + delete: + deletedConversationSettings.length > 0 + ? { + ids: deletedConversationSettings, + } + : undefined, }); const didUpdateKnowledgeBase = @@ -156,6 +163,7 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { updatedSystemPromptSettings, http, updatedConversationSettings, + createdConversationSettings, deletedConversationSettings, knowledgeBase.isEnabledKnowledgeBase, knowledgeBase.isEnabledRAGAlerts, @@ -170,6 +178,8 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { return { conversationSettings: updatedConversationSettings, + createdConversationSettings, + deletedConversationSettings, defaultAllow: updatedDefaultAllow, defaultAllowReplacement: updatedDefaultAllowReplacement, knowledgeBase: updatedKnowledgeBaseSettings, @@ -180,6 +190,7 @@ export const useSettingsUpdater = (): UseSettingsUpdater => { setUpdatedDefaultAllow, setUpdatedDefaultAllowReplacement, setUpdatedConversationSettings, + setCreatedConversationSettings, setUpdatedKnowledgeBaseSettings, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index 34090d74dd6cc..a7b7ed7294411 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -14,6 +14,7 @@ import { } from '../schemas/conversations/common_attributes.gen'; import { transformToCreateScheme } from './create_conversation'; import { transformToUpdateScheme } from './update_conversation'; +import { SearchEsConversationSchema } from './types'; interface WriterBulkResponse { errors: string[]; @@ -55,7 +56,8 @@ export class ConversationDataWriter implements ConversationDataWriter { } const { errors, items, took } = await this.options.esClient.bulk({ - operations: this.buildBulkOperations(params), + refresh: 'wait_for', + body: await this.buildBulkOperations(params), }); return { @@ -87,23 +89,45 @@ export class ConversationDataWriter implements ConversationDataWriter { } }; - private buildBulkOperations = (params: BulkParams): BulkOperationContainer[] => { + private buildBulkOperations = async (params: BulkParams): Promise => { const changedAt = new Date().toISOString(); const conversationBody = params.conversationsToCreate?.flatMap((conversation) => [ - { create: { _index: this.options.index } }, + { create: { _index: this.options.index, op_type: 'create' } }, transformToCreateScheme(changedAt, this.options.spaceId, this.options.user, conversation), ]) ?? []; const conversationUpdatedBody = params.conversationsToUpdate?.flatMap((conversation) => [ - { create: { _id: conversation.id, _index: this.options.index } }, + { update: { _id: conversation.id, _index: this.options.index } }, transformToUpdateScheme(changedAt, conversation), ]) ?? []; + const response = params.conversationsToDelete + ? await this.options.esClient.search({ + body: { + query: { + ids: { + values: params.conversationsToDelete, + }, + }, + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }) + : undefined; + const conversationDeletedBody = params.conversationsToDelete?.flatMap((conversationId) => [ - { create: { _id: conversationId, _index: this.options.index } }, + { + delete: { + _id: conversationId, + _index: response?.hits.hits.find((c) => c._id === conversationId)?._index, + }, + }, ]) ?? []; return [ diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index afdc827c1b9b6..e60b6cfd30032 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -69,11 +69,12 @@ export const createConversation = async ( conversation ); - const response = await esClient.create({ + const response = await esClient.index({ body, id: uuidv4(), index: conversationIndex, refresh: 'wait_for', + op_type: 'create', }); return { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 9f35548cd03cb..54dd0a3d5e352 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -66,8 +66,7 @@ export class AIAssistantConversationsDataClient { if (this.writerCache.get(spaceId)) { return this.writerCache.get(spaceId) as ConversationDataWriter; } - const indexPatterns = this.indexTemplateAndPattern; - await this.initializeWriter(spaceId, indexPatterns.alias); + await this.initializeWriter(spaceId, this.indexTemplateAndPattern.alias); return this.writerCache.get(spaceId) as ConversationDataWriter; } diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts index 5e6fbefb77e13..473e7d57e12f5 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts @@ -124,10 +124,10 @@ export const bulkActionConversationsRoute = ( }, async (context, request, response): Promise> => { const { body } = request; - const siemResponse = buildResponse(response); + const assistantResponse = buildResponse(response); if (body?.update && body.update?.length > CONVERSATIONS_TABLE_MAX_PAGE_SIZE) { - return siemResponse.error({ + return assistantResponse.error({ body: `More than ${CONVERSATIONS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, statusCode: 400, }); @@ -143,7 +143,6 @@ export const bulkActionConversationsRoute = ( const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const writer = await dataClient?.getWriter(); - const { errors, docs_created: docsCreated, @@ -177,7 +176,7 @@ export const bulkActionConversationsRoute = ( }); } catch (err) { const error = transformError(err); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); From 7311ef67035c132d6d81d8b5a7db13251ac86063 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 17 Jan 2024 17:35:38 -0800 Subject: [PATCH 015/141] updated data stream adapter --- packages/kbn-data-stream-adapter/README.md | 24 ++-- packages/kbn-data-stream-adapter/ecs.ts | 9 -- packages/kbn-data-stream-adapter/index.ts | 3 + packages/kbn-data-stream-adapter/package.json | 2 +- ...reate_or_update_component_template.test.ts | 110 ++--------------- .../create_or_update_component_template.ts | 3 +- .../src/data_stream_adapter.ts | 103 +++++++--------- .../src/data_stream_spaces_adapter.ts | 115 ++++++++---------- .../src/retry_transient_es_errors.ts | 8 +- .../kbn-data-stream-adapter/tsconfig.json | 1 - .../conversation_selector/index.tsx | 26 ++-- .../impl/assistant/index.tsx | 27 ++-- .../impl/assistant/use_conversation/index.tsx | 42 +++---- .../use_conversation/sample_conversations.tsx | 18 +-- .../impl/assistant_context/types.tsx | 1 - .../connectorland/connector_setup/index.tsx | 102 +++++++--------- .../impl/mock/conversation.ts | 12 -- .../server/ai_assistant_service/index.ts | 14 +-- .../assistant/content/conversations/index.tsx | 15 +-- 19 files changed, 220 insertions(+), 415 deletions(-) delete mode 100644 packages/kbn-data-stream-adapter/ecs.ts diff --git a/packages/kbn-data-stream-adapter/README.md b/packages/kbn-data-stream-adapter/README.md index f3d869b3554a6..04a3d854aced7 100644 --- a/packages/kbn-data-stream-adapter/README.md +++ b/packages/kbn-data-stream-adapter/README.md @@ -1,16 +1,15 @@ # @kbn/data-stream-adapter -Utility library to for Elasticsearch data stream creation +Utility library for Elasticsearch data stream management. ## DataStreamAdapter -It creates a single data stream. Example: +Manage single data streams. Example: ``` -// Instantiate +// Setup const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); -// Define component and index templates dataStream.setComponentTemplate({ name: 'awesome-component-template', fieldMap: { @@ -30,20 +29,19 @@ dataStream.setIndexTemplate({ }, }); -// Installs templates and data stream, or updates existing. -await dataStream.install({ logger, esClient, pluginStop$ }); +// Start +await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing. ``` ## DataStreamSpacesAdapter -It creates space aware data streams. Example: +Manage data streams per space. Example: ``` -// Instantiate +// Setup const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); -// Define component and index templates spacesDataStream.setComponentTemplate({ name: 'awesome-component-template', fieldMap: { @@ -63,9 +61,9 @@ spacesDataStream.setIndexTemplate({ }, }); -// Installs templates and updates existing data streams. -await spacesDataStream.install({ logger, esClient, pluginStop$ }); +// Start +await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams. -// After installation we can create space-aware data streams on runtime. -await spacesDataStream.installSpace('space2'); // creates `my-awesome-datastream-space2` if it does not exist +// Create a space data stream on the fly +await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist. ``` diff --git a/packages/kbn-data-stream-adapter/ecs.ts b/packages/kbn-data-stream-adapter/ecs.ts deleted file mode 100644 index c8993d7f1b5e7..0000000000000 --- a/packages/kbn-data-stream-adapter/ecs.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export * from './src/field_maps/ecs_field_map'; diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts index c9e3bd0950d58..808145be4f12e 100644 --- a/packages/kbn-data-stream-adapter/index.ts +++ b/packages/kbn-data-stream-adapter/index.ts @@ -8,6 +8,9 @@ export { DataStreamAdapter } from './src/data_stream_adapter'; export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter'; +export { retryTransientEsErrors } from './src/retry_transient_es_errors'; +export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map'; + export type { DataStreamAdapterParams, SetComponentTemplateParams, diff --git a/packages/kbn-data-stream-adapter/package.json b/packages/kbn-data-stream-adapter/package.json index 3f9118a5916bf..80b16c25ac217 100644 --- a/packages/kbn-data-stream-adapter/package.json +++ b/packages/kbn-data-stream-adapter/package.json @@ -1,7 +1,7 @@ { "name": "@kbn/data-stream-adapter", "version": "1.0.0", - "description": "Utility library to for Elasticsearch DataStream creation", + "description": "Utility library for Elasticsearch Data Stream management", "license": "SSPL-1.0 OR Elastic License 2.0", "private": true } diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts index a20156163d6b6..3bd93b6bbcb08 100644 --- a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts @@ -7,9 +7,9 @@ */ import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import type { DiagnosticResult } from '@elastic/elasticsearch'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; const randomDelayMultiplier = 0.01; const logger = loggingSystemMock.createLogger(); @@ -107,39 +107,9 @@ describe('createOrUpdateComponentTemplate', () => { it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => { clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { - error: { - root_cause: [ - { - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - }, - ], - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', - caused_by: { - type: 'illegal_argument_exception', - reason: 'Limit of total fields [1900] has been exceeded', - }, - }, - }, - }, - }, - }) - ) + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', + } as DiagnosticResult) ); const existingIndexTemplate = { name: 'test-template', @@ -193,39 +163,9 @@ describe('createOrUpdateComponentTemplate', () => { it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => { clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { - error: { - root_cause: [ - { - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - }, - ], - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', - caused_by: { - type: 'illegal_argument_exception', - reason: 'Limit of total fields [1900] has been exceeded', - }, - }, - }, - }, - }, - }) - ) + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', + } as DiagnosticResult) ); const existingIndexTemplate = { name: 'test-template', @@ -299,39 +239,9 @@ describe('createOrUpdateComponentTemplate', () => { it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => { clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { - error: { - root_cause: [ - { - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - }, - ], - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', - caused_by: { - type: 'illegal_argument_exception', - reason: 'Limit of total fields [1900] has been exceeded', - }, - }, - }, - }, - }, - }) - ) + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', + } as DiagnosticResult) ); const existingIndexTemplate = { name: 'test-template', diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts index f3c2e55d5569e..9e6a1f2f788dd 100644 --- a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts @@ -70,8 +70,7 @@ const createOrUpdateComponentTemplateHelper = async ( try { await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger }); } catch (error) { - const reason = error?.meta?.body?.error?.caused_by?.caused_by?.caused_by?.reason; - if (reason && reason.match(/Limit of total fields \[\d+\] has been exceeded/) != null) { + if (error.message.match(/Limit of total fields \[\d+\] has been exceeded/) != null) { // This error message occurs when there is an index template using this component template // that contains a field limit setting that using this component template exceeds // Specifically, this can happen for the ECS component template when we add new fields diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts index d2f824971c14b..3b3e2958eb46a 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts @@ -46,19 +46,11 @@ export interface GetInstallFnParams { } export interface InstallParams { logger: Logger; - esClient: ElasticsearchClient; + esClient: ElasticsearchClient | Promise; pluginStop$: Subject; tasksTimeoutMs?: number; } -export interface InstallationPromise { - result: boolean; - error?: string; -} - -export const successResult = () => ({ result: true }); -export const errorResult = (error?: string) => ({ result: false, error }); - const DEFAULT_FIELDS_LIMIT = 2500; export class DataStreamAdapter { @@ -114,58 +106,55 @@ export class DataStreamAdapter { }; } - public async install({ logger, esClient, pluginStop$, tasksTimeoutMs }: InstallParams) { - if (this.installed) { - throw new Error('Cannot re-install data stream'); - } - try { - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + public async install({ + logger, + esClient: esClientToResolve, + pluginStop$, + tasksTimeoutMs, + }: InstallParams) { + this.installed = true; + + const esClient = await esClientToResolve; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `${componentTemplate.name} component template` - ) + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${componentTemplate.name} component template` ) - ); + ) + ); - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ - template: indexTemplate, - esClient, - logger, - }), - `${indexTemplate.name} index template` - ) + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ + template: indexTemplate, + esClient, + logger, + }), + `${indexTemplate.name} index template` ) - ); + ) + ); - // create data stream when everything is ready - await installFn( - createOrUpdateDataStream({ - name: this.name, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `${this.name} data stream` - ); - this.installed = true; - return successResult(); - } catch (error) { - logger.error(`Error initializing data stream resources: ${error.message}`); - this.installed = false; - return errorResult(error.message); - } + // create data stream when everything is ready + await installFn( + createOrUpdateDataStream({ + name: this.name, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${this.name} data stream` + ); } } diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts index 95c497d62e54a..5daad080d4720 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts @@ -10,11 +10,8 @@ import { createDataStream, updateDataStreams } from './create_or_update_data_str import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; import { DataStreamAdapter, - InstallationPromise, type DataStreamAdapterParams, type InstallParams, - successResult, - errorResult, } from './data_stream_adapter'; export class DataStreamSpacesAdapter extends DataStreamAdapter { @@ -28,76 +25,66 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter { public async install({ logger, - esClient, + esClient: esClientToResolve, pluginStop$, tasksTimeoutMs, - }: InstallParams): Promise { - if (this.installed) { - throw new Error('Cannot re-install data stream'); - } + }: InstallParams) { + this.installed = true; - try { - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + const esClient = await esClientToResolve; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `create or update ${componentTemplate.name} component template` - ) + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `create or update ${componentTemplate.name} component template` ) - ); + ) + ); - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), - `create or update ${indexTemplate.name} index template` - ) + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), + `create or update ${indexTemplate.name} index template` ) - ); - - // Update existing space data streams - await installFn( - updateDataStreams({ - name: `${this.prefix}-*`, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `update space data streams` - ); + ) + ); - // define function to install data stream for spaces on demand - this._installSpace = async (spaceId: string) => { - const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); - if (existingInstallPromise) { - return existingInstallPromise; - } - const name = `${this.prefix}-${spaceId}`; - const installPromise = installFn( - createDataStream({ name, esClient, logger }), - `create ${name} data stream` - ).then(() => name); + // Update existing space data streams + await installFn( + updateDataStreams({ + name: `${this.prefix}-*`, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `update space data streams` + ); - this.installedSpaceDataStreamName.set(spaceId, installPromise); - return installPromise; - }; + // define function to install data stream for spaces on demand + this._installSpace = async (spaceId: string) => { + const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); + if (existingInstallPromise) { + return existingInstallPromise; + } + const name = `${this.prefix}-${spaceId}`; + const installPromise = installFn( + createDataStream({ name, esClient, logger }), + `create ${name} data stream` + ).then(() => name); - this.installed = true; - return successResult(); - } catch (error) { - logger.error(`Error initializing data stream resources: ${error.message}`); - this.installed = false; - return errorResult(error.message); - } + this.installedSpaceDataStreamName.set(spaceId, installPromise); + return installPromise; + }; } public async installSpace(spaceId: string): Promise { @@ -107,7 +94,7 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter { return this._installSpace(spaceId); } - public async getSpaceIndexName(spaceId: string): Promise { + public async getInstalledSpaceName(spaceId: string): Promise { return this.installedSpaceDataStreamName.get(spaceId); } } diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts index 893c477223ca3..3b436298e5c8d 100644 --- a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts +++ b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts @@ -29,13 +29,7 @@ const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const retryTransientEsErrors = async ( esCall: () => Promise, - { - logger, - attempt = 0, - }: { - logger: Logger; - attempt?: number; - } + { logger, attempt = 0 }: { logger: Logger; attempt?: number } ): Promise => { try { return await esCall(); diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json index 4fdd3065124fb..f09d2b4354d02 100644 --- a/packages/kbn-data-stream-adapter/tsconfig.json +++ b/packages/kbn-data-stream-adapter/tsconfig.json @@ -14,7 +14,6 @@ "include": ["**/*.ts", "**/*.tsx"], "kbn_references": [ "@kbn/core", - "@kbn/core-elasticsearch-client-server-mocks", "@kbn/std", "@kbn/ecs", "@kbn/alerts-as-data-utils", diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 572fe9b0a2385..519192f756406 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -33,7 +33,7 @@ interface Props { defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; selectedConversationId: string | undefined; - onConversationSelected: (conversationId: string) => void; + onConversationSelected: (conversationId: string, title?: string) => void; onConversationDeleted: (conversationId: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; @@ -70,7 +70,6 @@ export const ConversationSelector: React.FC = React.memo( const { allSystemPrompts } = useAssistantContext(); const { createConversation } = useConversation(); - const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); const conversationOptions = useMemo(() => { return Object.values(conversations).map((conversation) => ({ @@ -130,11 +129,20 @@ export const ConversationSelector: React.FC = React.memo( const onDelete = useCallback( (cId: string) => { if (selectedConversationId === cId) { - onConversationSelected(getPreviousConversationId(conversationIds, cId)); + onConversationSelected( + getPreviousConversationId(conversationIds, cId), + conversations[cId].title + ); } onConversationDeleted(cId); }, - [selectedConversationId, onConversationDeleted, onConversationSelected, conversationIds] + [ + selectedConversationId, + onConversationDeleted, + onConversationSelected, + conversationIds, + conversations, + ] ); const onChange = useCallback( @@ -142,7 +150,7 @@ export const ConversationSelector: React.FC = React.memo( if (newOptions.length === 0 || !newOptions?.[0].id) { setSelectedOptions([]); } else if (conversationOptions.findIndex((o) => o.id === newOptions?.[0].id) !== -1) { - await onConversationSelected(newOptions?.[0].id); + await onConversationSelected(newOptions?.[0].id, newOptions?.[0].label); } }, [conversationOptions, onConversationSelected] @@ -150,12 +158,12 @@ export const ConversationSelector: React.FC = React.memo( const onLeftArrowClick = useCallback(() => { const prevId = getPreviousConversationId(conversationIds, selectedConversationId); - onConversationSelected(prevId); - }, [conversationIds, selectedConversationId, onConversationSelected]); + onConversationSelected(prevId, conversations[prevId].title); + }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); const onRightArrowClick = useCallback(() => { const nextId = getNextConversationId(conversationIds, selectedConversationId); - onConversationSelected(nextId); - }, [conversationIds, selectedConversationId, onConversationSelected]); + onConversationSelected(nextId, conversations[nextId].title); + }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); // Register keyboard listener for quick conversation switching const onKeyDown = useCallback( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 56004c55089d2..970fa67a48f8f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -230,11 +230,10 @@ const AssistantComponent: React.FC = ({ const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ conversation: blockBotConversation, + conversations, + setConversations, }); - const currentTitle: string | JSX.Element = - isWelcomeSetup && blockBotConversation.theme?.title ? blockBotConversation.theme?.title : title; - const [promptTextPreview, setPromptTextPreview] = useState(''); const [autoPopulatedOnce, setAutoPopulatedOnce] = useState(false); const [userPrompt, setUserPrompt] = useState(null); @@ -301,7 +300,7 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - async (cId: string) => { + async (cId: string, cTitle?: string) => { if (conversations[cId] === undefined && cId) { const updatedConv = await refetchResults(); if (updatedConv) { @@ -311,24 +310,24 @@ const AssistantComponent: React.FC = ({ getDefaultSystemPrompt({ allSystemPrompts, conversation: updatedConv[cId] })?.id ); } - } else if (cId) { + } else if (cId && cId !== cTitle) { setSelectedConversationId(cId); const refetchedConversation = await refetchCurrentConversation(cId); if (refetchedConversation) { setCurrentConversation(refetchedConversation); } setEditingSystemPromptId( - getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation })?.id + getDefaultSystemPrompt({ allSystemPrompts, conversation: refetchedConversation })?.id + ); + } else { + setSelectedConversationId(cId); + setCurrentConversation(conversations[cId]); + setEditingSystemPromptId( + getDefaultSystemPrompt({ allSystemPrompts, conversation: conversations[cId] })?.id ); } }, - [ - allSystemPrompts, - conversations, - currentConversation, - refetchCurrentConversation, - refetchResults, - ] + [allSystemPrompts, conversations, refetchCurrentConversation, refetchResults] ); const handleOnConversationDeleted = useCallback( @@ -558,7 +557,7 @@ const AssistantComponent: React.FC = ({ setIsSettingsModalVisible={setIsSettingsModalVisible} setSelectedConversationId={setSelectedConversationId} showAnonymizedValues={showAnonymizedValues} - title={currentTitle} + title={title} conversations={conversations} onConversationDeleted={handleOnConversationDeleted} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index ef7101f92fedd..6ac67fb022899 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -11,7 +11,6 @@ import { IHttpFetchError, isHttpFetchError } from '@kbn/core-http-browser'; import { useAssistantContext } from '../../assistant_context'; import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; -import { ELASTIC_AI_ASSISTANT, ELASTIC_AI_ASSISTANT_TITLE } from './translations'; import { getDefaultSystemPrompt } from './helpers'; import { createConversationApi, @@ -19,25 +18,13 @@ import { getConversationById, updateConversationApi, } from '../api/conversations'; +import { WELCOME_CONVERSATION } from './sample_conversations'; export const DEFAULT_CONVERSATION_STATE: Conversation = { id: i18n.DEFAULT_CONVERSATION_TITLE, messages: [], apiConfig: {}, title: i18n.DEFAULT_CONVERSATION_TITLE, - theme: { - title: ELASTIC_AI_ASSISTANT_TITLE, - titleIcon: 'logoSecurity', - assistant: { - name: ELASTIC_AI_ASSISTANT, - icon: 'logoSecurity', - }, - system: { - icon: 'logoElastic', - }, - user: {}, - }, - user: {}, }; interface AppendMessageProps { @@ -83,6 +70,8 @@ interface UseConversation { setApiConfig: ({ conversationId, apiConfig, + title, + isDefault, }: SetApiConfigProps) => Promise>; createConversation: (conversation: Conversation) => Promise; getConversation: (conversationId: string) => Promise; @@ -247,15 +236,18 @@ export const useConversation = (): UseConversation => { conversation: undefined, })?.id; - const newConversation: Conversation = { - ...DEFAULT_CONVERSATION_STATE, - apiConfig: { - ...DEFAULT_CONVERSATION_STATE.apiConfig, - defaultSystemPromptId, - }, - id: conversationId, - messages: messages != null ? messages : [], - }; + const newConversation: Conversation = + conversationId === i18n.WELCOME_CONVERSATION_TITLE + ? WELCOME_CONVERSATION + : { + ...DEFAULT_CONVERSATION_STATE, + apiConfig: { + ...DEFAULT_CONVERSATION_STATE.apiConfig, + defaultSystemPromptId, + }, + id: conversationId, + messages: messages != null ? messages : [], + }; return newConversation; }, [allSystemPrompts] @@ -285,11 +277,11 @@ export const useConversation = (): UseConversation => { ); /** - * Update the apiConfig for a given conversationId + * Create/Update the apiConfig for a given conversationId */ const setApiConfig = useCallback( async ({ conversationId, apiConfig, title, isDefault }: SetApiConfigProps) => { - if (isDefault && title === conversationId) { + if (title === conversationId) { return createConversationApi({ http, conversation: { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx index e88981569a933..a3898e1edb9e9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx @@ -7,27 +7,11 @@ import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from '../../content/prompts/welcome/translations'; -import { - ELASTIC_AI_ASSISTANT, - ELASTIC_AI_ASSISTANT_TITLE, - WELCOME_CONVERSATION_TITLE, -} from './translations'; +import { WELCOME_CONVERSATION_TITLE } from './translations'; export const WELCOME_CONVERSATION: Conversation = { id: WELCOME_CONVERSATION_TITLE, title: WELCOME_CONVERSATION_TITLE, - theme: { - title: ELASTIC_AI_ASSISTANT_TITLE, - titleIcon: 'logoSecurity', - assistant: { - name: ELASTIC_AI_ASSISTANT, - icon: 'logoSecurity', - }, - system: { - icon: 'logoElastic', - }, - user: {}, - }, messages: [ { role: 'assistant', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 09f99d55440ae..568b659bb1ebf 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -67,7 +67,6 @@ export interface Conversation { messages: Message[]; updatedAt?: string; replacements?: Record; - theme?: ConversationTheme; isDefault?: boolean; excludeFromLastConversationStorage?: boolean; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 4026cab6923ae..7bae68ef0bc49 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import React, { useCallback, useMemo, useRef, useState } from 'react'; import type { EuiCommentProps } from '@elastic/eui'; import { EuiAvatar, EuiBadge, EuiMarkdownFormat, EuiText, EuiTextAlign } from '@elastic/eui'; // eslint-disable-next-line @kbn/eslint/module_migration @@ -13,15 +13,14 @@ import styled from 'styled-components'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public/common/constants'; import { ActionType } from '@kbn/triggers-actions-ui-plugin/public'; -import { merge } from 'lodash/fp'; import { AddConnectorModal } from '../add_connector_modal'; import { WELCOME_CONVERSATION } from '../../assistant/use_conversation/sample_conversations'; -import { Conversation, Message, useFetchCurrentUserConversations } from '../../..'; +import { Conversation, Message } from '../../..'; import { useLoadActionTypes } from '../use_load_action_types'; import { StreamingText } from '../../assistant/streaming_text'; import { ConnectorButton } from '../connector_button'; import { useConversation } from '../../assistant/use_conversation'; -import { conversationHasNoPresentationData } from './helpers'; +import { clearPresentationData, conversationHasNoPresentationData } from './helpers'; import * as i18n from '../translations'; import { useAssistantContext } from '../../assistant_context'; import { useLoadConnectors } from '../use_load_connectors'; @@ -39,47 +38,23 @@ const SkipEuiText = styled(EuiText)` export interface ConnectorSetupProps { conversation?: Conversation; onSetupComplete?: () => void; + conversations: Record; + setConversations: React.Dispatch>>; } export const useConnectorSetup = ({ conversation = WELCOME_CONVERSATION, onSetupComplete, + conversations, + setConversations, }: ConnectorSetupProps): { comments: EuiCommentProps[]; prompt: React.ReactElement; } => { - const [conversations, setConversations] = useState>({}); const { appendMessage, setApiConfig } = useConversation(); const bottomRef = useRef(null); // Access all conversations so we can add connector to all on initial setup - const { actionTypeRegistry, http, baseConversations } = useAssistantContext(); - - const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); - - useEffect(() => { - if (!isLoading) { - const userConversations = (conversationsData?.data ?? []).reduce< - Record - >((transformed, conversationData) => { - transformed[conversationData.id] = conversationData; - return transformed; - }, {}); - setConversations( - merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversationData) => { - transformed[conversationData] = baseConversations[conversationData]; - return transformed; - }, {}) - ) - ); - } - }, [baseConversations, conversationsData?.data, isLoading]); + const { actionTypeRegistry, http } = useAssistantContext(); const { data: connectors, @@ -97,15 +72,6 @@ export const useConnectorSetup = ({ const [selectedActionType, setSelectedActionType] = useState(null); - // User constants - const userName = useMemo( - () => conversation.theme?.user?.name ?? i18n.CONNECTOR_SETUP_USER_YOU, - [conversation.theme?.user?.name] - ); - const assistantName = useMemo( - () => conversation.theme?.assistant?.name ?? i18n.CONNECTOR_SETUP_USER_ASSISTANT, - [conversation.theme?.assistant?.name] - ); const lastConversationMessageIndex = useMemo( () => conversation.messages.length - 1, [conversation.messages.length] @@ -138,8 +104,8 @@ export const useConnectorSetup = ({ setShowAddConnectorButton(true); bottomRef.current?.scrollIntoView({ block: 'end' }); onSetupComplete?.(); - // setConversation({ conversation: clearPresentationData(conversation) }); - }, [onSetupComplete]); + setConversations({ ...conversations, [conversation.id]: clearPresentationData(conversation) }); + }, [conversation, conversations, onSetupComplete, setConversations]); // Show button to add connector after last message has finished streaming const handleSkipSetup = useCallback(() => { @@ -190,7 +156,7 @@ export const useConnectorSetup = ({ const isUser = message.role === 'user'; const commentProps: EuiCommentProps = { - username: isUser ? userName : assistantName, + username: isUser ? i18n.CONNECTOR_SETUP_USER_YOU : i18n.CONNECTOR_SETUP_USER_ASSISTANT, children: commentBody(message, index, conversation.messages.length), timelineAvatar: ( { - setApiConfig({ - conversationId: c.id, - title: c.title, - isDefault: c.isDefault, - apiConfig: { - ...c.apiConfig, - connectorId: connector.id, - connectorTypeTitle, - provider: config?.apiProvider, - model: config?.defaultModel, - }, - }); + // persist only the active conversation + + setApiConfig({ + conversationId: conversation.id, + title: conversation.title, + isDefault: conversation.isDefault, + apiConfig: { + ...conversation.apiConfig, + connectorId: connector.id, + connectorTypeTitle, + provider: config?.apiProvider, + model: config?.defaultModel, + }, }); + setConversations( + Object.values(conversations).reduce((res, c) => { + res[c.id] = { + ...c, + apiConfig: { + ...c.apiConfig, + connectorId: connector.id, + connectorTypeTitle, + provider: config?.apiProvider, + model: config?.defaultModel, + }, + }; + return res; + }, {} as Record) + ); refetchConnectors?.(); setIsConnectorModalVisible(false); @@ -241,10 +222,11 @@ export const useConnectorSetup = ({ [ actionTypeRegistry, appendMessage, - conversation.id, + conversation, conversations, refetchConnectors, setApiConfig, + setConversations, ] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts index 657f95f5a21c1..23ecacc7d9cf7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts @@ -35,18 +35,6 @@ export const emptyWelcomeConvo: Conversation = { id: 'Welcome', title: 'Welcome', isDefault: true, - theme: { - title: 'Elastic AI Assistant', - titleIcon: 'logoSecurity', - assistant: { - name: 'Elastic AI Assistant', - icon: 'logoSecurity', - }, - system: { - icon: 'logoElastic', - }, - user: {}, - }, messages: [], apiConfig: { connectorId: 'c29c28a0-20fe-11ee-9306-a1f4d42ec542', diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index da41a99ffe556..d2ca3d4c28f2a 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -101,24 +101,20 @@ export class AIAssistantService { this.options.logger.debug(`Initializing resources for AIAssistantService`); const esClient = await this.options.elasticsearchClientPromise; - const installationResult = await this.conversationsDataStream.install({ + await this.conversationsDataStream.install({ esClient, logger: this.options.logger, pluginStop$: this.options.pluginStop$, }); - - if (installationResult.error !== undefined) { - throw installationResult.error; - } - this.initialized = true; - this.isInitializing = false; - return successResult(); } catch (error) { this.options.logger.error(`Error initializing AI assistant resources: ${error.message}`); this.initialized = false; this.isInitializing = false; return errorResult(error.message); } + this.initialized = true; + this.isInitializing = false; + return successResult(); } private readonly resourceNames: AssistantResourceNames = { @@ -222,7 +218,7 @@ export class AIAssistantService { ) { try { this.options.logger.debug(`Initializing spaceId level resources for AIAssistantService`); - let indexName = await this.conversationsDataStream.getSpaceIndexName(spaceId); + let indexName = await this.conversationsDataStream.getInstalledSpaceName(spaceId); if (!indexName) { indexName = await this.conversationsDataStream.installSpace(spaceId); } diff --git a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx index 7d53b0c39d3e6..2216eb04fe294 100644 --- a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { ELASTIC_AI_ASSISTANT_TITLE, WELCOME_CONVERSATION_TITLE } from '@kbn/elastic-assistant'; +import { WELCOME_CONVERSATION_TITLE } from '@kbn/elastic-assistant'; import type { Conversation } from '@kbn/elastic-assistant'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; import { DETECTION_RULES_CONVERSATION_ID } from '../../../detections/pages/detection_engine/rules/translations'; @@ -13,7 +13,6 @@ import { ALERT_SUMMARY_CONVERSATION_ID, EVENT_SUMMARY_CONVERSATION_ID, } from '../../../common/components/event_details/translations'; -import { ELASTIC_AI_ASSISTANT } from '../../comment_actions/translations'; import { TIMELINE_CONVERSATION_TITLE } from './translations'; export const BASE_SECURITY_CONVERSATIONS: Record = { @@ -57,18 +56,6 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { id: WELCOME_CONVERSATION_TITLE, title: WELCOME_CONVERSATION_TITLE, isDefault: true, - theme: { - title: ELASTIC_AI_ASSISTANT_TITLE, - titleIcon: 'logoSecurity', - assistant: { - name: ELASTIC_AI_ASSISTANT, - icon: 'logoSecurity', - }, - system: { - icon: 'logoElastic', - }, - user: {}, - }, messages: [], apiConfig: {}, }, From ef9bd99cfaef0a7234fb0f5925ebcdaee8a8d4d7 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 17 Jan 2024 20:04:46 -0800 Subject: [PATCH 016/141] fixed frontend bulk api --- .../use_bulk_actions_conversations.ts | 62 ++++- .../impl/assistant/assistant_header/index.tsx | 2 +- .../conversation_settings.tsx | 245 +++++++++++------- .../system_prompt_settings.tsx | 78 +++++- .../assistant/settings/assistant_settings.tsx | 33 ++- .../settings/assistant_settings_button.tsx | 6 +- .../use_settings_updater.tsx | 77 ++---- 7 files changed, 315 insertions(+), 188 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 91cbf6e13f4f4..ad318a3e9bfc6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -5,13 +5,13 @@ * 2.0. */ -// import { AI_ASSISTANT_API_CURRENT_VERSION } from '../common/constants'; +import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { HttpSetup } from '@kbn/core/public'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; -import { Conversation } from '../../../assistant_context/types'; +import { Conversation, Message } from '../../../assistant_context/types'; export interface BulkActionSummary { failed: number; @@ -46,19 +46,61 @@ export interface BulkUpdateResponse { attributes: BulkActionAttributes; } +interface ConversationUpdateParams { + id?: string; + title?: string; + messages?: Message[]; + apiConfig?: { + connectorId?: string; + connectorTypeTitle?: string; + defaultSystemPromptId?: string; + provider?: OpenAiProviderType; + model?: string; + }; +} + +export interface ConversationsBulkActions { + update?: Record; + create?: Record; + delete?: { + ids: string[]; + }; +} + export const bulkConversationsChange = ( http: HttpSetup, - conversationsActions: { - update?: Array>; - create?: Conversation[]; - delete?: { - ids: string[]; - }; - } + conversationsActions: ConversationsBulkActions ) => { return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { method: 'POST', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - body: JSON.stringify(conversationsActions), + body: JSON.stringify({ + update: conversationsActions.update + ? Object.keys(conversationsActions.update).reduce( + (conversationsToUpdate: ConversationUpdateParams[], conversationId) => { + if (conversationsActions.update) { + conversationsToUpdate.push({ + id: conversationId, + ...conversationsActions.update[conversationId], + }); + } + return conversationsToUpdate; + }, + [] + ) + : undefined, + create: conversationsActions.create + ? Object.keys(conversationsActions.create).reduce( + (conversationsToCreate: Conversation[], conversationId: string) => { + if (conversationsActions.create) { + conversationsToCreate.push(conversationsActions.create[conversationId]); + } + return conversationsToCreate; + }, + [] + ) + : undefined, + delete: conversationsActions.delete, + }), }); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index 2fd7e57c3fe70..39b9783d1c9b4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -138,7 +138,7 @@ export const AssistantHeader: React.FC = ({ isSettingsModalVisible={isSettingsModalVisible} selectedConversation={currentConversation} setIsSettingsModalVisible={setIsSettingsModalVisible} - setSelectedConversationId={setSelectedConversationId} + onConversationSelected={onConversationSelected} conversations={conversations} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index 41311ce04d14f..2d481b9c8abe5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -23,23 +23,20 @@ import { ConversationSelectorSettings } from '../conversation_selector_settings' import { getDefaultSystemPrompt } from '../../use_conversation/helpers'; import { useLoadConnectors } from '../../../connectorland/use_load_connectors'; import { getGenAiConfig } from '../../../connectorland/helpers'; +import { ConversationsBulkActions } from '../../api'; export interface ConversationSettingsProps { allSystemPrompts: Prompt[]; conversationSettings: Record; - createdConversationSettings: Record; - deletedConversationSettings: string[]; + conversationsSettingsBulkActions: ConversationsBulkActions; defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; http: HttpSetup; onSelectedConversationChange: (conversation?: Conversation) => void; selectedConversation: Conversation | undefined; - setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction> - >; - setDeletedConversationSettings: React.Dispatch>; - setCreatedConversationSettings: React.Dispatch< - React.SetStateAction> + setConversationSettings: React.Dispatch>>; + setConversationsSettingsBulkActions: React.Dispatch< + React.SetStateAction >; isDisabled?: boolean; } @@ -55,13 +52,11 @@ export const ConversationSettings: React.FC = React.m selectedConversation, onSelectedConversationChange, conversationSettings, - createdConversationSettings, - deletedConversationSettings, http, - setUpdatedConversationSettings, isDisabled = false, - setCreatedConversationSettings, - setDeletedConversationSettings, + setConversationSettings, + conversationsSettingsBulkActions, + setConversationsSettingsBulkActions, }) => { const defaultSystemPrompt = useMemo(() => { return getDefaultSystemPrompt({ allSystemPrompts, conversation: undefined }); @@ -95,12 +90,19 @@ export const ConversationSettings: React.FC = React.m newSelectedConversation && (isNew || newSelectedConversation.id === newSelectedConversation.title) ) { - setCreatedConversationSettings({ - ...createdConversationSettings, + setConversationSettings({ + ...conversationSettings, [isNew ? c : newSelectedConversation.id]: newSelectedConversation, }); + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + create: { + ...(conversationsSettingsBulkActions.create ?? {}), + [newSelectedConversation.id]: newSelectedConversation, + }, + }); } else if (newSelectedConversation != null) { - setUpdatedConversationSettings((prev) => { + setConversationSettings((prev) => { return { ...prev, [newSelectedConversation.id]: newSelectedConversation, @@ -111,57 +113,88 @@ export const ConversationSettings: React.FC = React.m onSelectedConversationChange(newSelectedConversation); }, [ - createdConversationSettings, + conversationSettings, + conversationsSettingsBulkActions, defaultConnectorId, defaultProvider, defaultSystemPrompt?.id, onSelectedConversationChange, - setCreatedConversationSettings, - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, ] ); const onConversationDeleted = useCallback( (conversationId: string) => { - setDeletedConversationSettings([...deletedConversationSettings, conversationId]); + const updatedConverationSettings = { ...conversationSettings }; + delete updatedConverationSettings[conversationId]; + setConversationSettings(updatedConverationSettings); + + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + delete: { + ids: [...(conversationsSettingsBulkActions.delete?.ids ?? []), conversationId], + }, + }); }, - [deletedConversationSettings, setDeletedConversationSettings] + [ + conversationSettings, + conversationsSettingsBulkActions, + setConversationSettings, + setConversationsSettingsBulkActions, + ] ); const handleOnSystemPromptSelectionChange = useCallback( (systemPromptId?: string | undefined) => { if (selectedConversation != null) { - if (conversationSettings[selectedConversation.id]) { - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - defaultSystemPromptId: systemPromptId, + const updatedConversation = { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + defaultSystemPromptId: systemPromptId, + }, + }; + setConversationSettings({ + ...conversationSettings, + [updatedConversation.id]: updatedConversation, + }); + if (selectedConversation.id !== selectedConversation.title) { + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + update: { + ...(conversationsSettingsBulkActions.update ?? {}), + [updatedConversation.id]: { + ...(conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {}), + apiConfig: { + ...((conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {} + ).apiConfig ?? {}), + defaultSystemPromptId: systemPromptId, + }, }, - updatedAt: undefined, }, - })); + }); } else { - setCreatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - defaultSystemPromptId: systemPromptId, - }, + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + create: { + ...(conversationsSettingsBulkActions.create ?? {}), + [updatedConversation.id]: updatedConversation, }, - })); + }); } } }, [ conversationSettings, + conversationsSettingsBulkActions, selectedConversation, - setCreatedConversationSettings, - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, ] ); @@ -182,42 +215,56 @@ export const ConversationSettings: React.FC = React.m (connector) => { if (selectedConversation != null) { const config = getGenAiConfig(connector); - - if (conversationSettings[selectedConversation.id]) { - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - connectorId: connector?.id, - provider: config?.apiProvider, - model: config?.defaultModel, + const updatedConversation = { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + connectorId: connector?.id, + provider: config?.apiProvider, + model: config?.defaultModel, + }, + }; + setConversationSettings((prev) => ({ + ...prev, + [selectedConversation.id]: updatedConversation, + })); + if (selectedConversation.id !== selectedConversation.title) { + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + update: { + ...(conversationsSettingsBulkActions.update ?? {}), + [updatedConversation.id]: { + ...(conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {}), + apiConfig: { + ...((conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {} + ).apiConfig ?? {}), + connectorId: connector?.id, + provider: config?.apiProvider, + model: config?.defaultModel, + }, }, - updatedAt: undefined, }, - })); + }); } else { - setCreatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - connectorId: connector?.id, - provider: config?.apiProvider, - model: config?.defaultModel, - }, + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + create: { + ...(conversationsSettingsBulkActions.create ?? {}), + [updatedConversation.id]: updatedConversation, }, - })); + }); } } }, [ - conversationSettings, + conversationsSettingsBulkActions, selectedConversation, - setCreatedConversationSettings, - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, ] ); @@ -230,37 +277,53 @@ export const ConversationSettings: React.FC = React.m const handleOnModelSelectionChange = useCallback( (model?: string) => { if (selectedConversation != null) { - if (conversationSettings[selectedConversation.id]) { - setUpdatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - model, + const updatedConversation = { + ...selectedConversation, + apiConfig: { + ...selectedConversation.apiConfig, + model, + }, + }; + setConversationSettings({ + ...conversationSettings, + [updatedConversation.id]: updatedConversation, + }); + if (selectedConversation.id !== selectedConversation.title) { + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + update: { + ...(conversationsSettingsBulkActions.update ?? {}), + [updatedConversation.id]: { + ...(conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {}), + apiConfig: { + ...((conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[updatedConversation.id] ?? {} + : {} + ).apiConfig ?? {}), + model, + }, }, - updatedAt: undefined, }, - })); + }); } else { - setCreatedConversationSettings((prev) => ({ - ...prev, - [selectedConversation.id]: { - ...selectedConversation, - apiConfig: { - ...selectedConversation.apiConfig, - model, - }, + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + create: { + ...(conversationsSettingsBulkActions.create ?? {}), + [updatedConversation.id]: updatedConversation, }, - })); + }); } } }, [ conversationSettings, + conversationsSettingsBulkActions, selectedConversation, - setCreatedConversationSettings, - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, ] ); @@ -275,7 +338,7 @@ export const ConversationSettings: React.FC = React.m diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx index 91317f3d2436e..6b2a358765783 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx @@ -27,16 +27,19 @@ import * as i18n from './translations'; import { ConversationMultiSelector } from './conversation_multi_selector/conversation_multi_selector'; import { SystemPromptSelector } from './system_prompt_selector/system_prompt_selector'; import { TEST_IDS } from '../../../constants'; +import { ConversationsBulkActions } from '../../../api'; interface Props { conversationSettings: Record; + conversationsSettingsBulkActions: ConversationsBulkActions; onSelectedSystemPromptChange: (systemPrompt?: Prompt) => void; selectedSystemPrompt: Prompt | undefined; setUpdatedSystemPromptSettings: React.Dispatch>; - setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction> - >; + setConversationSettings: React.Dispatch>>; systemPromptSettings: Prompt[]; + setConversationsSettingsBulkActions: React.Dispatch< + React.SetStateAction + >; } /** @@ -48,8 +51,10 @@ export const SystemPromptSettings: React.FC = React.memo( onSelectedSystemPromptChange, selectedSystemPrompt, setUpdatedSystemPromptSettings, - setUpdatedConversationSettings, + setConversationSettings, systemPromptSettings, + conversationsSettingsBulkActions, + setConversationsSettingsBulkActions, }) => { // Prompt const promptContent = useMemo( @@ -101,9 +106,9 @@ export const SystemPromptSettings: React.FC = React.memo( const currentPromptConversationIds = currentPromptConversations.map((convo) => convo.id); if (selectedSystemPrompt != null) { - setUpdatedConversationSettings((prev) => + setConversationSettings((prev) => keyBy( - 'title', + 'id', /* * updatedConversationWithPrompts calculates the present of prompt for * each conversation. Based on the values of selected conversation, it goes @@ -127,9 +132,68 @@ export const SystemPromptSettings: React.FC = React.memo( })) ) ); + + Object.values(conversationSettings).forEach((convo) => { + if (convo.id !== convo.title) { + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + update: { + ...(conversationsSettingsBulkActions.update ?? {}), + [convo.id]: { + ...(conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[convo.id] ?? {} + : {}), + apiConfig: { + ...((conversationsSettingsBulkActions.update + ? conversationsSettingsBulkActions.update[convo.id] ?? {} + : {} + ).apiConfig ?? {}), + defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) + ? selectedSystemPrompt?.id + : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id + ? // remove the default System Prompt if it is assigned to a conversation + // but that conversation is not in the currentPromptConversationList + // This means conversation was removed in the current transaction + undefined + : // leave it as it is .. if that conversation was neither added nor removed. + convo.apiConfig.defaultSystemPromptId, + }, + }, + }, + }); + } else { + setConversationsSettingsBulkActions({ + ...conversationsSettingsBulkActions, + create: { + ...(conversationsSettingsBulkActions.create ?? {}), + [convo.id]: { + ...convo, + apiConfig: { + ...convo.apiConfig, + defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) + ? selectedSystemPrompt?.id + : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id + ? // remove the default System Prompt if it is assigned to a conversation + // but that conversation is not in the currentPromptConversationList + // This means conversation was removed in the current transaction + undefined + : // leave it as it is .. if that conversation was neither added nor removed. + convo.apiConfig.defaultSystemPromptId, + }, + }, + }, + }); + } + }); } }, - [selectedSystemPrompt, setUpdatedConversationSettings] + [ + conversationSettings, + conversationsSettingsBulkActions, + selectedSystemPrompt, + setConversationSettings, + setConversationsSettingsBulkActions, + ] ); // Whether this system prompt should be the default for new conversations diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index e34eb983eb8b9..06ca75ccb32e3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -65,7 +65,7 @@ interface Props { ) => void; onSave: () => void; selectedConversation: Conversation; - setSelectedConversationId: React.Dispatch>; + onConversationSelected: (cId: string) => void; conversations: Record; } @@ -80,7 +80,7 @@ export const AssistantSettings: React.FC = React.memo( onClose, onSave, selectedConversation: defaultSelectedConversation, - setSelectedConversationId, + onConversationSelected, conversations, }) => { const { modelEvaluatorEnabled, http, selectedSettingsTab, setSelectedSettingsTab } = @@ -88,22 +88,20 @@ export const AssistantSettings: React.FC = React.memo( const { conversationSettings, + setConversationSettings, defaultAllow, defaultAllowReplacement, - deletedConversationSettings, knowledgeBase, quickPromptSettings, systemPromptSettings, - setUpdatedConversationSettings, setUpdatedDefaultAllow, setUpdatedDefaultAllowReplacement, setUpdatedKnowledgeBaseSettings, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, saveSettings, - createdConversationSettings, - setCreatedConversationSettings, - setDeletedConversationSettings, + conversationsSettingsBulkActions, + setConversationsSettingsBulkActions, } = useSettingsUpdater(conversations); // Local state for saving previously selected items so tab switching is friendlier @@ -118,10 +116,9 @@ export const AssistantSettings: React.FC = React.memo( }, []); useEffect(() => { if (selectedConversation != null) { - const v = conversationSettings[selectedConversation.id]; - setSelectedConversation(v ? v : createdConversationSettings[selectedConversation.id]); + setSelectedConversation(conversationSettings[selectedConversation.id]); } - }, [conversationSettings, createdConversationSettings, selectedConversation]); + }, [conversationSettings, selectedConversation]); // Quick Prompt Selection State const [selectedQuickPrompt, setSelectedQuickPrompt] = useState(); @@ -153,16 +150,16 @@ export const AssistantSettings: React.FC = React.memo( conversationSettings[defaultSelectedConversation.id] == null; const newSelectedConversationId: string | undefined = Object.keys(conversationSettings)[0]; if (isSelectedConversationDeleted && newSelectedConversationId != null) { - setSelectedConversationId(conversationSettings[newSelectedConversationId].id); + onConversationSelected(newSelectedConversationId); } saveSettings(); onSave(); }, [ conversationSettings, defaultSelectedConversation.id, + onConversationSelected, onSave, saveSettings, - setSelectedConversationId, ]); return ( @@ -289,11 +286,9 @@ export const AssistantSettings: React.FC = React.memo( defaultConnectorId={defaultConnectorId} defaultProvider={defaultProvider} conversationSettings={conversationSettings} - createdConversationSettings={createdConversationSettings} - deletedConversationSettings={deletedConversationSettings} - setUpdatedConversationSettings={setUpdatedConversationSettings} - setCreatedConversationSettings={setCreatedConversationSettings} - setDeletedConversationSettings={setDeletedConversationSettings} + setConversationsSettingsBulkActions={setConversationsSettingsBulkActions} + conversationsSettingsBulkActions={conversationsSettingsBulkActions} + setConversationSettings={setConversationSettings} allSystemPrompts={systemPromptSettings} selectedConversation={selectedConversation} isDisabled={selectedConversation == null} @@ -315,7 +310,9 @@ export const AssistantSettings: React.FC = React.memo( systemPromptSettings={systemPromptSettings} onSelectedSystemPromptChange={onHandleSelectedSystemPromptChange} selectedSystemPrompt={selectedSystemPrompt} - setUpdatedConversationSettings={setUpdatedConversationSettings} + setConversationSettings={setConversationSettings} + setConversationsSettingsBulkActions={setConversationsSettingsBulkActions} + conversationsSettingsBulkActions={conversationsSettingsBulkActions} setUpdatedSystemPromptSettings={setUpdatedSystemPromptSettings} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index 40a3251d18cd6..6d0b17b5e8f4c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -20,7 +20,7 @@ interface Props { isSettingsModalVisible: boolean; selectedConversation: Conversation; setIsSettingsModalVisible: React.Dispatch>; - setSelectedConversationId: React.Dispatch>; + onConversationSelected: (cId: string) => void; isDisabled?: boolean; conversations: Record; } @@ -36,7 +36,7 @@ export const AssistantSettingsButton: React.FC = React.memo( isSettingsModalVisible, setIsSettingsModalVisible, selectedConversation, - setSelectedConversationId, + onConversationSelected, conversations, }) => { const { toasts, setSelectedSettingsTab } = useAssistantContext(); @@ -81,7 +81,7 @@ export const AssistantSettingsButton: React.FC = React.memo( defaultConnectorId={defaultConnectorId} defaultProvider={defaultProvider} selectedConversation={selectedConversation} - setSelectedConversationId={setSelectedConversationId} + onConversationSelected={onConversationSelected} onClose={handleCloseModal} onSave={handleSave} conversations={conversations} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 81f8c78cbe518..62d2af9ca3f5d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -9,12 +9,14 @@ import React, { useCallback, useState } from 'react'; import { Conversation, Prompt, QuickPrompt } from '../../../..'; import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; -import { bulkConversationsChange } from '../../api/conversations/use_bulk_actions_conversations'; +import { + ConversationsBulkActions, + bulkConversationsChange, +} from '../../api/conversations/use_bulk_actions_conversations'; interface UseSettingsUpdater { conversationSettings: Record; - createdConversationSettings: Record; - deletedConversationSettings: string[]; + conversationsSettingsBulkActions: ConversationsBulkActions; defaultAllow: string[]; defaultAllowReplacement: string[]; knowledgeBase: KnowledgeBaseConfig; @@ -23,16 +25,13 @@ interface UseSettingsUpdater { systemPromptSettings: Prompt[]; setUpdatedDefaultAllow: React.Dispatch>; setUpdatedDefaultAllowReplacement: React.Dispatch>; - setUpdatedConversationSettings: React.Dispatch< - React.SetStateAction> - >; - setCreatedConversationSettings: React.Dispatch< - React.SetStateAction> + setConversationSettings: React.Dispatch>>; + setConversationsSettingsBulkActions: React.Dispatch< + React.SetStateAction >; setUpdatedKnowledgeBaseSettings: React.Dispatch>; setUpdatedQuickPromptSettings: React.Dispatch>; setUpdatedSystemPromptSettings: React.Dispatch>; - setDeletedConversationSettings: React.Dispatch>; saveSettings: () => void; } @@ -59,12 +58,10 @@ export const useSettingsUpdater = ( * Pending updating state */ // Conversations - const [updatedConversationSettings, setUpdatedConversationSettings] = + const [conversationSettings, setConversationSettings] = useState>(conversations); - const [createdConversationSettings, setCreatedConversationSettings] = useState< - Record - >({}); - const [deletedConversationSettings, setDeletedConversationSettings] = useState([]); + const [conversationsSettingsBulkActions, setConversationsSettingsBulkActions] = + useState({}); // Quick Prompts const [updatedQuickPromptSettings, setUpdatedQuickPromptSettings] = useState(allQuickPrompts); @@ -83,9 +80,8 @@ export const useSettingsUpdater = ( * Reset all pending settings */ const resetSettings = useCallback((): void => { - setUpdatedConversationSettings(conversations); - setDeletedConversationSettings([]); - setCreatedConversationSettings({}); + setConversationSettings(conversations); + setConversationsSettingsBulkActions({}); setUpdatedQuickPromptSettings(allQuickPrompts); setUpdatedKnowledgeBaseSettings(knowledgeBase); setUpdatedSystemPromptSettings(allSystemPrompts); @@ -106,38 +102,7 @@ export const useSettingsUpdater = ( const saveSettings = useCallback((): void => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - bulkConversationsChange(http, { - update: Object.keys(updatedConversationSettings).reduce( - ( - conversationsToUpdate: Array>, - conversationId: string - ) => { - if (updatedConversationSettings.updatedAt === undefined) { - conversationsToUpdate.push({ - ...(updatedConversationSettings[conversationId] as Omit< - Conversation, - 'createdAt' | 'updatedAt' | 'user' - >), - }); - } - return conversationsToUpdate; - }, - [] - ), - create: Object.keys(createdConversationSettings).reduce( - (conversationsToCreate: Conversation[], conversationId: string) => { - conversationsToCreate.push(createdConversationSettings[conversationId]); - return conversationsToCreate; - }, - [] - ), - delete: - deletedConversationSettings.length > 0 - ? { - ids: deletedConversationSettings, - } - : undefined, - }); + bulkConversationsChange(http, conversationsSettingsBulkActions); const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; @@ -162,9 +127,7 @@ export const useSettingsUpdater = ( setAllSystemPrompts, updatedSystemPromptSettings, http, - updatedConversationSettings, - createdConversationSettings, - deletedConversationSettings, + conversationsSettingsBulkActions, knowledgeBase.isEnabledKnowledgeBase, knowledgeBase.isEnabledRAGAlerts, updatedKnowledgeBaseSettings, @@ -177,9 +140,8 @@ export const useSettingsUpdater = ( ]); return { - conversationSettings: updatedConversationSettings, - createdConversationSettings, - deletedConversationSettings, + conversationSettings, + conversationsSettingsBulkActions, defaultAllow: updatedDefaultAllow, defaultAllowReplacement: updatedDefaultAllowReplacement, knowledgeBase: updatedKnowledgeBaseSettings, @@ -189,11 +151,10 @@ export const useSettingsUpdater = ( saveSettings, setUpdatedDefaultAllow, setUpdatedDefaultAllowReplacement, - setUpdatedConversationSettings, - setCreatedConversationSettings, setUpdatedKnowledgeBaseSettings, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, - setDeletedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, }; }; From 582d94751f340a02991b96699f9c37f1ce1db633 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 17 Jan 2024 20:48:44 -0800 Subject: [PATCH 017/141] fixed bulk update api --- .../conversations_data_writer.ts | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index a7b7ed7294411..b6f589380da1c 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -97,10 +97,61 @@ export class ConversationDataWriter implements ConversationDataWriter { transformToCreateScheme(changedAt, this.options.spaceId, this.options.user, conversation), ]) ?? []; + const updatedAt = new Date().toISOString(); const conversationUpdatedBody = params.conversationsToUpdate?.flatMap((conversation) => [ { update: { _id: conversation.id, _index: this.options.index } }, - transformToUpdateScheme(changedAt, conversation), + { + script: { + source: ` + if (params.assignEmpty == true || params.containsKey('api_config')) { + if (params.assignEmpty == true || params.api_config.containsKey('connector_id')) { + ctx._source.api_config.connector_id = params.api_config.connector_id; + } + if (params.assignEmpty == true || params.api_config.containsKey('connector_type_title')) { + ctx._source.api_config.connector_type_title = params.api_config.connector_type_title; + } + if (params.assignEmpty == true || params.api_config.containsKey('default_system_prompt_id')) { + ctx._source.api_config.default_system_prompt_id = params.api_config.default_system_prompt_id; + } + if (params.assignEmpty == true || params.api_config.containsKey('model')) { + ctx._source.api_config.model = params.api_config.model; + } + if (params.assignEmpty == true || params.api_config.containsKey('provider')) { + ctx._source.api_config.provider = params.api_config.provider; + } + } + if (params.assignEmpty == true || params.containsKey('exclude_from_last_conversation_storage')) { + ctx._source.exclude_from_last_conversation_storage = params.exclude_from_last_conversation_storage; + } + if (params.assignEmpty == true || params.containsKey('replacements')) { + ctx._source.replacements = params.replacements; + } + if (params.assignEmpty == true || params.containsKey('title')) { + ctx._source.title = params.title; + } + if (params.assignEmpty == true || params.containsKey('messages')) { + def messages = []; + for (message in params.messages) { + def newMessage = [:]; + newMessage['@timestamp'] = message['@timestamp']; + newMessage.content = message.content; + newMessage.is_error = message.is_error; + newMessage.presentation = message.presentation; + newMessage.reader = message.reader; + newMessage.replacements = message.replacements; + newMessage.role = message.role; + messages.add(newMessage); + } + ctx._source.messages = messages; + } + ctx._source.updated_at = params.updated_at; + `, + lang: 'painless', + params: transformToUpdateScheme(updatedAt, conversation), + }, + upsert: { counter: 1 }, + }, ]) ?? []; const response = params.conversationsToDelete From 8bada676ab842940241559260a74d3aec21b5979 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 18 Jan 2024 11:19:17 -0800 Subject: [PATCH 018/141] fixed system prompt change to do the bulk update on save --- .../impl/assistant/assistant_header/index.tsx | 5 ++- .../impl/assistant/index.tsx | 25 +++++++----- .../conversation_multi_selector.tsx | 7 ++-- .../system_prompt_settings.tsx | 26 +++++++------ .../assistant/settings/assistant_settings.tsx | 6 +-- .../settings/assistant_settings_button.tsx | 7 +++- .../use_settings_updater.tsx | 6 +-- .../conversations_data_writer.ts | 38 +++++++++++++++++-- 8 files changed, 83 insertions(+), 37 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index 39b9783d1c9b4..b1f35b47701b7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -35,12 +35,12 @@ interface OwnProps { onConversationDeleted: (conversationId: string) => void; onToggleShowAnonymizedValues: (e: EuiSwitchEvent) => void; setIsSettingsModalVisible: React.Dispatch>; - setSelectedConversationId: React.Dispatch>; setCurrentConversation: React.Dispatch>; shouldDisableKeyboardShortcut?: () => boolean; showAnonymizedValues: boolean; title: string | JSX.Element; conversations: Record; + refetchConversationsState: () => Promise; } type Props = OwnProps; @@ -60,12 +60,12 @@ export const AssistantHeader: React.FC = ({ onConversationDeleted, onToggleShowAnonymizedValues, setIsSettingsModalVisible, - setSelectedConversationId, shouldDisableKeyboardShortcut, showAnonymizedValues, title, setCurrentConversation, conversations, + refetchConversationsState, }) => { const showAnonymizedValuesChecked = useMemo( () => @@ -140,6 +140,7 @@ export const AssistantHeader: React.FC = ({ setIsSettingsModalVisible={setIsSettingsModalVisible} onConversationSelected={onConversationSelected} conversations={conversations} + refetchConversationsState={refetchConversationsState} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 970fa67a48f8f..558ca6d9cfe90 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -104,7 +104,18 @@ const AssistantComponent: React.FC = ({ const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = useConversation(); const { data: conversationsData, isLoading, refetch } = useFetchCurrentUserConversations(); - const { data: lastConversation, isLoading: isLoadingLast } = useLastConversation(); + const { + data: lastConversation, + isLoading: isLoadingLast, + refetch: refetchLastUpdated, + } = useLastConversation(); + + const lastConversationId = useMemo(() => { + if (!isLoadingLast) { + return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; + } + return WELCOME_CONVERSATION_TITLE; + }, [isLoadingLast, lastConversation?.id]); useEffect(() => { if (!isLoading) { @@ -169,13 +180,6 @@ const AssistantComponent: React.FC = ({ [connectors] ); - const lastConversationId = useMemo(() => { - if (!isLoadingLast) { - return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; - } - return WELCOME_CONVERSATION_TITLE; - }, [isLoadingLast, lastConversation?.id]); - const [selectedConversationId, setSelectedConversationId] = useState( isAssistantEnabled ? lastConversationId : WELCOME_CONVERSATION_TITLE ); @@ -555,11 +559,14 @@ const AssistantComponent: React.FC = ({ onConversationSelected={handleOnConversationSelected} onToggleShowAnonymizedValues={onToggleShowAnonymizedValues} setIsSettingsModalVisible={setIsSettingsModalVisible} - setSelectedConversationId={setSelectedConversationId} showAnonymizedValues={showAnonymizedValues} title={title} conversations={conversations} onConversationDeleted={handleOnConversationDeleted} + refetchConversationsState={async () => { + await refetchResults(); + await refetchCurrentConversation(); + }} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx index 5e5dfaa5831e0..d13c8146621cf 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx @@ -33,7 +33,8 @@ export const ConversationMultiSelector: React.FC = React.memo( const options = useMemo( () => conversations.map((conversation) => ({ - label: conversation.id ?? '', + // id: conversation.id + label: conversation.title ?? '', 'data-test-subj': TEST_IDS.CONVERSATIONS_MULTISELECTOR_OPTION(conversation.id), })), [conversations] @@ -41,7 +42,7 @@ export const ConversationMultiSelector: React.FC = React.memo( const selectedOptions = useMemo(() => { return selectedConversations != null ? selectedConversations.map((conversation) => ({ - label: conversation.id, + label: conversation.title, })) : []; }, [selectedConversations]); @@ -49,7 +50,7 @@ export const ConversationMultiSelector: React.FC = React.memo( const handleSelectionChange = useCallback( (conversationMultiSelectorOption: EuiComboBoxOptionOption[]) => { const newConversationSelection = conversations.filter((conversation) => - conversationMultiSelectorOption.some((cmso) => conversation.id === cmso.label) + conversationMultiSelectorOption.some((cmso) => conversation.title === cmso.label) ); onConversationSelectionChange(newConversationSelection); }, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx index 6b2a358765783..545956627de92 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx @@ -133,19 +133,20 @@ export const SystemPromptSettings: React.FC = React.memo( ) ); + let updatedConversationsSettingsBulkActions = { ...conversationsSettingsBulkActions }; Object.values(conversationSettings).forEach((convo) => { if (convo.id !== convo.title) { - setConversationsSettingsBulkActions({ - ...conversationsSettingsBulkActions, + updatedConversationsSettingsBulkActions = { + ...updatedConversationsSettingsBulkActions, update: { - ...(conversationsSettingsBulkActions.update ?? {}), + ...(updatedConversationsSettingsBulkActions.update ?? {}), [convo.id]: { - ...(conversationsSettingsBulkActions.update - ? conversationsSettingsBulkActions.update[convo.id] ?? {} + ...(updatedConversationsSettingsBulkActions.update + ? updatedConversationsSettingsBulkActions.update[convo.id] ?? {} : {}), apiConfig: { - ...((conversationsSettingsBulkActions.update - ? conversationsSettingsBulkActions.update[convo.id] ?? {} + ...((updatedConversationsSettingsBulkActions.update + ? updatedConversationsSettingsBulkActions.update[convo.id] ?? {} : {} ).apiConfig ?? {}), defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) @@ -160,12 +161,12 @@ export const SystemPromptSettings: React.FC = React.memo( }, }, }, - }); + }; } else { - setConversationsSettingsBulkActions({ - ...conversationsSettingsBulkActions, + updatedConversationsSettingsBulkActions = { + ...updatedConversationsSettingsBulkActions, create: { - ...(conversationsSettingsBulkActions.create ?? {}), + ...(updatedConversationsSettingsBulkActions.create ?? {}), [convo.id]: { ...convo, apiConfig: { @@ -182,9 +183,10 @@ export const SystemPromptSettings: React.FC = React.memo( }, }, }, - }); + }; } }); + setConversationsSettingsBulkActions(updatedConversationsSettingsBulkActions); } }, [ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index 06ca75ccb32e3..a3ed3d66e56ea 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -63,7 +63,7 @@ interface Props { onClose: ( event?: React.KeyboardEvent | React.MouseEvent ) => void; - onSave: () => void; + onSave: () => Promise; selectedConversation: Conversation; onConversationSelected: (cId: string) => void; conversations: Record; @@ -144,7 +144,7 @@ export const AssistantSettings: React.FC = React.memo( } }, [selectedSystemPrompt, systemPromptSettings]); - const handleSave = useCallback(() => { + const handleSave = useCallback(async () => { // If the selected conversation is deleted, we need to select a new conversation to prevent a crash creating a conversation that already exists const isSelectedConversationDeleted = conversationSettings[defaultSelectedConversation.id] == null; @@ -152,7 +152,7 @@ export const AssistantSettings: React.FC = React.memo( if (isSelectedConversationDeleted && newSelectedConversationId != null) { onConversationSelected(newSelectedConversationId); } - saveSettings(); + await saveSettings(); onSave(); }, [ conversationSettings, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index 6d0b17b5e8f4c..4d294c1b123fb 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -23,6 +23,7 @@ interface Props { onConversationSelected: (cId: string) => void; isDisabled?: boolean; conversations: Record; + refetchConversationsState: () => Promise; } /** @@ -38,6 +39,7 @@ export const AssistantSettingsButton: React.FC = React.memo( selectedConversation, onConversationSelected, conversations, + refetchConversationsState, }) => { const { toasts, setSelectedSettingsTab } = useAssistantContext(); @@ -50,13 +52,14 @@ export const AssistantSettingsButton: React.FC = React.memo( cleanupAndCloseModal(); }, [cleanupAndCloseModal]); - const handleSave = useCallback(() => { + const handleSave = useCallback(async () => { cleanupAndCloseModal(); + await refetchConversationsState(); toasts?.addSuccess({ iconType: 'check', title: i18n.SETTINGS_UPDATED_TOAST_TITLE, }); - }, [cleanupAndCloseModal, toasts]); + }, [cleanupAndCloseModal, refetchConversationsState, toasts]); const handleShowConversationSettings = useCallback(() => { setSelectedSettingsTab(CONVERSATIONS_TAB); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 62d2af9ca3f5d..53ed76ff5dae6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -32,7 +32,7 @@ interface UseSettingsUpdater { setUpdatedKnowledgeBaseSettings: React.Dispatch>; setUpdatedQuickPromptSettings: React.Dispatch>; setUpdatedSystemPromptSettings: React.Dispatch>; - saveSettings: () => void; + saveSettings: () => Promise; } export const useSettingsUpdater = ( @@ -99,10 +99,10 @@ export const useSettingsUpdater = ( /** * Save all pending settings */ - const saveSettings = useCallback((): void => { + const saveSettings = useCallback(async (): Promise => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - bulkConversationsChange(http, conversationsSettingsBulkActions); + await bulkConversationsChange(http, conversationsSettingsBulkActions); const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index b6f589380da1c..c183ef623e453 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -28,6 +28,7 @@ interface BulkParams { conversationsToCreate?: ConversationCreateProps[]; conversationsToUpdate?: ConversationUpdateProps[]; conversationsToDelete?: string[]; + isPatch?: boolean; } export interface ConversationDataWriter { @@ -63,7 +64,12 @@ export class ConversationDataWriter implements ConversationDataWriter { return { errors: errors ? items - .map((item) => item.create?.error?.reason) + .map( + (item) => + item.create?.error?.reason ?? + item.update?.error?.reason ?? + item.delete?.error?.reason + ) .filter((error): error is string => !!error) : [], docs_created: items @@ -98,9 +104,31 @@ export class ConversationDataWriter implements ConversationDataWriter { ]) ?? []; const updatedAt = new Date().toISOString(); + + const responseToUpdate = params.conversationsToUpdate + ? await this.options.esClient.search({ + body: { + query: { + ids: { + values: params.conversationsToUpdate?.map((c) => c.id), + }, + }, + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }) + : undefined; const conversationUpdatedBody = params.conversationsToUpdate?.flatMap((conversation) => [ - { update: { _id: conversation.id, _index: this.options.index } }, + { + update: { + _id: conversation.id, + _index: responseToUpdate?.hits.hits.find((c) => c._id === conversation.id)?._index, + }, + }, { script: { source: ` @@ -148,7 +176,11 @@ export class ConversationDataWriter implements ConversationDataWriter { ctx._source.updated_at = params.updated_at; `, lang: 'painless', - params: transformToUpdateScheme(updatedAt, conversation), + params: { + ...transformToUpdateScheme(updatedAt, conversation), // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !(params.isPatch ?? true), + }, }, upsert: { counter: 1 }, }, From dd4008a99d80d64959ef22f160fa491b33eadbf8 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 18 Jan 2024 12:07:57 -0800 Subject: [PATCH 019/141] added migration from the local storage --- .../packages/kbn-elastic-assistant/index.ts | 1 + .../public/assistant/provider.tsx | 33 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 3aa191fa478d5..9c7ad660cdde6 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -142,3 +142,4 @@ export type { GetKnowledgeBaseStatusResponse } from './impl/assistant/api'; export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; export { useFetchCurrentUserConversations } from './impl/assistant/api/conversations/use_fetch_current_user_conversations'; +export * from './impl/assistant/api/conversations/use_bulk_actions_conversations'; diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 5b5f60f322c8c..9d8b0e482cd77 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -7,12 +7,16 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; import type { IToasts } from '@kbn/core-notifications-browser'; -import { AssistantProvider as ElasticAssistantProvider } from '@kbn/elastic-assistant'; +import type { Conversation } from '@kbn/elastic-assistant'; +import { + AssistantProvider as ElasticAssistantProvider, + bulkConversationsChange, +} from '@kbn/elastic-assistant'; import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; -import { augmentMessageCodeBlocks } from './helpers'; +import { LOCAL_STORAGE_KEY, augmentMessageCodeBlocks } from './helpers'; import { useBaseConversations } from './use_conversation_store'; import { DEFAULT_ALLOW, DEFAULT_ALLOW_REPLACEMENT } from './content/anonymization'; import { PROMPT_CONTEXTS } from './content/prompt_contexts'; @@ -22,6 +26,7 @@ import { useAnonymizationStore } from './use_anonymization_store'; import { useAssistantAvailability } from './use_assistant_availability'; import { useAppToasts } from '../common/hooks/use_app_toasts'; import { useSignalIndex } from '../detections/containers/detection_engine/alerts/use_signal_index'; +import { useLocalStorage } from '../common/components/local_storage'; const ASSISTANT_TITLE = i18n.translate('xpack.securitySolution.assistant.title', { defaultMessage: 'Elastic AI Assistant', @@ -33,6 +38,7 @@ const ASSISTANT_TITLE = i18n.translate('xpack.securitySolution.assistant.title', export const AssistantProvider: React.FC = ({ children }) => { const { http, + storage, triggersActionsUi: { actionTypeRegistry }, docLinks: { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION }, } = useKibana().services; @@ -49,6 +55,29 @@ export const AssistantProvider: React.FC = ({ children }) => { const alertsIndexPattern = signalIndexName ?? undefined; const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) + // migrate used conversations from the local storage + const [conversations, setConversations] = useLocalStorage< + Record | undefined + >({ + defaultValue: undefined, + key: LOCAL_STORAGE_KEY, + }); + + if (conversations && Object.keys(conversations).length > 0) { + const conversationsToCreate = Object.values(conversations).filter( + (c) => c.messages && c.messages.length > 0 + ); + // post bulk create + bulkConversationsChange(http, { + create: conversationsToCreate.reduce((res: Record, c) => { + res[c.id] = c; + return res; + }, {}), + }); + setConversations(undefined); + storage.remove(LOCAL_STORAGE_KEY); + } + return ( Date: Thu, 18 Jan 2024 12:34:50 -0800 Subject: [PATCH 020/141] fixed migration with title --- .../public/assistant/provider.tsx | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 9d8b0e482cd77..d6275975d3c18 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -26,7 +26,6 @@ import { useAnonymizationStore } from './use_anonymization_store'; import { useAssistantAvailability } from './use_assistant_availability'; import { useAppToasts } from '../common/hooks/use_app_toasts'; import { useSignalIndex } from '../detections/containers/detection_engine/alerts/use_signal_index'; -import { useLocalStorage } from '../common/components/local_storage'; const ASSISTANT_TITLE = i18n.translate('xpack.securitySolution.assistant.title', { defaultMessage: 'Elastic AI Assistant', @@ -55,27 +54,21 @@ export const AssistantProvider: React.FC = ({ children }) => { const alertsIndexPattern = signalIndexName ?? undefined; const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) - // migrate used conversations from the local storage - const [conversations, setConversations] = useLocalStorage< - Record | undefined - >({ - defaultValue: undefined, - key: LOCAL_STORAGE_KEY, - }); + // migrate conversations from the local storage if its have messages + const conversations = storage.get(`securitySolution.${LOCAL_STORAGE_KEY}`); if (conversations && Object.keys(conversations).length > 0) { - const conversationsToCreate = Object.values(conversations).filter( - (c) => c.messages && c.messages.length > 0 - ); + const conversationsToCreate = Object.values( + conversations as Record + ).filter((c) => c.messages && c.messages.length > 0); // post bulk create bulkConversationsChange(http, { create: conversationsToCreate.reduce((res: Record, c) => { - res[c.id] = c; + res[c.id] = { ...c, title: c.id }; return res; }, {}), }); - setConversations(undefined); - storage.remove(LOCAL_STORAGE_KEY); + storage.remove(`securitySolution.${LOCAL_STORAGE_KEY}`); } return ( From 8ccb28f7b02ce9d6321a687ff9163696a759402a Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Fri, 19 Jan 2024 16:08:48 -0800 Subject: [PATCH 021/141] fixed uncesarry Assistant rendering --- .../use_bulk_actions_conversations.ts | 12 +-- .../use_fetch_current_user_conversations.ts | 35 +++++--- .../impl/assistant/index.tsx | 89 +++++++++---------- .../use_settings_updater.tsx | 4 +- .../packages/kbn-elastic-assistant/index.ts | 1 + .../public/assistant/provider.tsx | 47 ++++++---- .../use_assistant_telemetry/index.tsx | 23 ++--- .../use_conversation_store/index.tsx | 55 +++++++----- .../timeline/assistant_tab_content/index.tsx | 35 ++++++++ .../timeline/tabs_content/index.tsx | 79 ++++++---------- 10 files changed, 212 insertions(+), 168 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/assistant_tab_content/index.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index ad318a3e9bfc6..2fb46215d3a4b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -31,7 +31,7 @@ export interface BulkActionAggregatedError { message: string; status_code: number; err_code?: string; - rules: Array<{ id: string; name?: string }>; + conversations: Array<{ id: string; name?: string }>; } export interface BulkActionAttributes { @@ -40,9 +40,11 @@ export interface BulkActionAttributes { errors?: BulkActionAggregatedError[]; } -export interface BulkUpdateResponse { +export interface BulkActionResponse { success?: boolean; - rules_count?: number; + conversations_count?: number; + message?: string; + status_code?: number; attributes: BulkActionAttributes; } @@ -67,11 +69,11 @@ export interface ConversationsBulkActions { }; } -export const bulkConversationsChange = ( +export const bulkChangeConversations = ( http: HttpSetup, conversationsActions: ConversationsBulkActions ) => { - return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { + return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { method: 'POST', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, body: JSON.stringify({ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts index 9d9670622ca73..08f3e65170f1e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts @@ -22,27 +22,34 @@ export interface FetchConversationsResponse { data: Conversation[]; } -export const useFetchCurrentUserConversations = () => { +export const useFetchCurrentUserConversations = ( + onFetch: (result: FetchConversationsResponse) => Record +) => { const { http } = useKibana().services; const query = { page: 1, perPage: 100, }; - const querySt = useQuery( - [ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, query], - () => - http.fetch( - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, - { - method: 'GET', - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - query, - } - ) - ); + const cachingKeys = [ + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + query.page, + query.perPage, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ]; + const querySt = useQuery([cachingKeys, query], async () => { + const res = await http.fetch( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + { + method: 'GET', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + query, + } + ); + return onFetch(res); + }); - return { ...querySt }; + return querySt; }; /** diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 558ca6d9cfe90..d8ec053a599a5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -51,6 +51,7 @@ import { useLoadConnectors } from '../connectorland/use_load_connectors'; import { useConnectorSetup } from '../connectorland/connector_setup'; import { ConnectorMissingCallout } from '../connectorland/connector_missing_callout'; import { + FetchConversationsResponse, useFetchCurrentUserConversations, useLastConversation, } from './api/conversations/use_fetch_current_user_conversations'; @@ -101,9 +102,44 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); + const onFetchedConversations = useCallback( + (conversationsData: FetchConversationsResponse): Record => { + const userConversations = (conversationsData?.data ?? []).reduce< + Record + >((transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, {}); + return merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ); + }, + [baseConversations] + ); + const { + data: conversationsData, + isLoading, + isError, + refetch, + } = useFetchCurrentUserConversations(onFetchedConversations); + const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = useConversation(); - const { data: conversationsData, isLoading, refetch } = useFetchCurrentUserConversations(); + useEffect(() => { + if (!isLoading && !isError) { + setConversations(conversationsData ?? {}); + } + }, [conversationsData, isError, isLoading]); + const { data: lastConversation, isLoading: isLoadingLast, @@ -117,54 +153,13 @@ const AssistantComponent: React.FC = ({ return WELCOME_CONVERSATION_TITLE; }, [isLoadingLast, lastConversation?.id]); - useEffect(() => { - if (!isLoading) { - const userConversations = (conversationsData?.data ?? []).reduce< - Record - >((transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, {}); - setConversations( - merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ) - ); - } - }, [baseConversations, conversationsData?.data, isLoading]); - const refetchResults = useCallback(async () => { - const res = await refetch(); - if (!res.isLoading) { - const userConversations = (res?.data?.data ?? []).reduce>( - (transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, - {} - ); - const updatedConv = merge( - userConversations, - Object.keys(baseConversations) - .filter((baseId) => (res.data?.data ?? []).find((c) => c.title === baseId) === undefined) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ); - setConversations(updatedConv); - return updatedConv; + const updatedConv = await refetch(); + if (!updatedConv.isLoading) { + setConversations(updatedConv.data ?? {}); + return updatedConv.data; } - }, [baseConversations, refetch]); + }, [refetch]); // Connector details const { data: connectors, isSuccess: areConnectorsFetched } = useLoadConnectors({ http }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 53ed76ff5dae6..1b161d1497331 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -11,7 +11,7 @@ import { useAssistantContext } from '../../../assistant_context'; import type { KnowledgeBaseConfig } from '../../types'; import { ConversationsBulkActions, - bulkConversationsChange, + bulkChangeConversations, } from '../../api/conversations/use_bulk_actions_conversations'; interface UseSettingsUpdater { @@ -102,7 +102,7 @@ export const useSettingsUpdater = ( const saveSettings = useCallback(async (): Promise => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - await bulkConversationsChange(http, conversationsSettingsBulkActions); + await bulkChangeConversations(http, conversationsSettingsBulkActions); const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 9c7ad660cdde6..8c79f87161408 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -143,3 +143,4 @@ export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; export { useFetchCurrentUserConversations } from './impl/assistant/api/conversations/use_fetch_current_user_conversations'; export * from './impl/assistant/api/conversations/use_bulk_actions_conversations'; +export { getConversationById } from './impl/assistant/api/conversations/conversations'; diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index d6275975d3c18..380a456fb9875 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -4,20 +4,20 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import type { IToasts } from '@kbn/core-notifications-browser'; import type { Conversation } from '@kbn/elastic-assistant'; import { AssistantProvider as ElasticAssistantProvider, - bulkConversationsChange, + bulkChangeConversations, } from '@kbn/elastic-assistant'; import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; import { LOCAL_STORAGE_KEY, augmentMessageCodeBlocks } from './helpers'; -import { useBaseConversations } from './use_conversation_store'; +import { useBaseConversations, useConversationStore } from './use_conversation_store'; import { DEFAULT_ALLOW, DEFAULT_ALLOW_REPLACEMENT } from './content/anonymization'; import { PROMPT_CONTEXTS } from './content/prompt_contexts'; import { BASE_SECURITY_QUICK_PROMPTS } from './content/quick_prompts'; @@ -44,6 +44,7 @@ export const AssistantProvider: React.FC = ({ children }) => { const basePath = useBasePath(); const baseConversations = useBaseConversations(); + const userConversations = useConversationStore(); const assistantAvailability = useAssistantAvailability(); const assistantTelemetry = useAssistantTelemetry(); @@ -54,22 +55,34 @@ export const AssistantProvider: React.FC = ({ children }) => { const alertsIndexPattern = signalIndexName ?? undefined; const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) - // migrate conversations from the local storage if its have messages + // migrate conversations with messages from the local storage + // won't happen again if the user conversations exist in the index const conversations = storage.get(`securitySolution.${LOCAL_STORAGE_KEY}`); - if (conversations && Object.keys(conversations).length > 0) { - const conversationsToCreate = Object.values( - conversations as Record - ).filter((c) => c.messages && c.messages.length > 0); - // post bulk create - bulkConversationsChange(http, { - create: conversationsToCreate.reduce((res: Record, c) => { - res[c.id] = { ...c, title: c.id }; - return res; - }, {}), - }); - storage.remove(`securitySolution.${LOCAL_STORAGE_KEY}`); - } + useEffect(() => { + const migrateConversationsFromLocalStorage = async () => { + if ( + Object.keys(userConversations).length > 0 && + conversations && + Object.keys(conversations).length > 0 + ) { + const conversationsToCreate = Object.values( + conversations as Record + ).filter((c) => c.messages && c.messages.length > 0); + // post bulk create + const bulkResult = await bulkChangeConversations(http, { + create: conversationsToCreate.reduce((res: Record, c) => { + res[c.id] = { ...c, title: c.id }; + return res; + }, {}), + }); + if (bulkResult.success) { + storage.remove(`securitySolution.${LOCAL_STORAGE_KEY}`); + } + } + }; + migrateConversationsFromLocalStorage(); + }, [conversations, http, storage, userConversations]); return ( { - const conversations = useConversationStore(); const { - services: { telemetry }, + services: { telemetry, http }, } = useKibana(); const getAnonymizedConversationId = useCallback( - (id) => { - const convo = conversations[id] ?? { isDefault: false }; + async (id) => { + const conversation = await getConversationById({ http, id }); + const convo = (conversation as Conversation) ?? { isDefault: false }; return convo.isDefault ? id : 'Custom'; }, - [conversations] + [http] ); const reportTelemetry = useCallback( - ({ + async ({ fn, params: { conversationId, ...rest }, - }): { fn: keyof AssistantTelemetry; params: AssistantTelemetry[keyof AssistantTelemetry] } => + }): Promise<{ + fn: keyof AssistantTelemetry; + params: AssistantTelemetry[keyof AssistantTelemetry]; + }> => fn({ ...rest, - conversationId: getAnonymizedConversationId(conversationId), + conversationId: await getAnonymizedConversationId(conversationId), }), [getAnonymizedConversationId] ); diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index 8bb563599eef6..5676759529a3d 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -9,7 +9,8 @@ import { useFetchCurrentUserConversations, type Conversation } from '@kbn/elasti import { merge, unset } from 'lodash/fp'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; -import { useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import type { FetchConversationsResponse } from '@kbn/elastic-assistant/impl/assistant/api'; import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { useLinkAuthorized } from '../../common/links'; import { SecurityPageName } from '../../../common'; @@ -25,37 +26,47 @@ export const useConversationStore = (): Record => { [isDataQualityDashboardPageExists] ); - const { data: conversationsData, isLoading } = useFetchCurrentUserConversations(); - - useEffect(() => { - if (!isLoading) { + const onFetchedConversations = useCallback( + (conversationsData: FetchConversationsResponse): Record => { const userConversations = (conversationsData?.data ?? []).reduce< Record >((transformed, conversation) => { transformed[conversation.id] = conversation; return transformed; }, {}); - - const notUsedBaseConversations = Object.keys(baseConversations).filter( - (baseId) => (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ); - - setConversations( - merge( - userConversations, - notUsedBaseConversations.reduce>( - (transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, - {} + return merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => + (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined ) - ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) ); + }, + [baseConversations] + ); + const { + data: conversationsData, + isLoading, + isError, + } = useFetchCurrentUserConversations(onFetchedConversations); + + useEffect(() => { + if (!isLoading && !isError) { + setConversations(conversationsData ?? {}); } - }, [baseConversations, conversationsData?.data, isLoading]); + }, [conversationsData, isLoading, isError]); + + const result = useMemo( + () => merge(baseConversations, conversations), + [baseConversations, conversations] + ); - return merge(baseConversations, conversations); + return result; }; export const useBaseConversations = (): Record => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/assistant_tab_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/assistant_tab_content/index.tsx new file mode 100644 index 0000000000000..5c4f31a67080d --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/assistant_tab_content/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import styled from 'styled-components'; +import { Assistant } from '@kbn/elastic-assistant'; +import type { Dispatch, SetStateAction } from 'react'; +import React, { memo } from 'react'; +import { TIMELINE_CONVERSATION_TITLE } from '../../../../assistant/content/conversations/translations'; + +const AssistantTabContainer = styled.div` + overflow-y: auto; + width: 100%; +`; + +const AssistantTab: React.FC<{ + shouldRefocusPrompt: boolean; + setConversationId: Dispatch>; +}> = memo(({ shouldRefocusPrompt, setConversationId }) => ( + + + +)); + +AssistantTab.displayName = 'AssistantTab'; + +// eslint-disable-next-line import/no-default-export +export { AssistantTab as default }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index d2c334684de86..0c9282da70aef 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -7,7 +7,6 @@ import { EuiBadge, EuiSkeletonText, EuiTabs, EuiTab, EuiBetaBadge } from '@elastic/eui'; import { css } from '@emotion/react'; -import { Assistant } from '@kbn/elastic-assistant'; import { isEmpty } from 'lodash/fp'; import type { Ref, ReactElement, ComponentType, Dispatch, SetStateAction } from 'react'; import React, { lazy, memo, Suspense, useCallback, useEffect, useMemo, useState } from 'react'; @@ -18,7 +17,6 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; import { useKibana } from '../../../../common/lib/kibana'; import { useAssistantTelemetry } from '../../../../assistant/use_assistant_telemetry'; -import { useConversationStore } from '../../../../assistant/use_conversation_store'; import { useAssistantAvailability } from '../../../../assistant/use_assistant_availability'; import type { SessionViewConfig } from '../../../../../common/types'; import type { RowRenderer, TimelineId } from '../../../../../common/types/timeline'; @@ -78,11 +76,6 @@ const tabWithSuspense =

( return Comp; }; -const AssistantTabContainer = styled.div` - overflow-y: auto; - width: 100%; -`; - const QueryTab = tabWithSuspense(lazy(() => import('../query_tab_content'))); const EqlTab = tabWithSuspense(lazy(() => import('../eql_tab_content'))); const GraphTab = tabWithSuspense(lazy(() => import('../graph_tab_content'))); @@ -90,6 +83,7 @@ const NotesTab = tabWithSuspense(lazy(() => import('../notes_tab_content'))); const PinnedTab = tabWithSuspense(lazy(() => import('../pinned_tab_content'))); const SessionTab = tabWithSuspense(lazy(() => import('../session_tab_content'))); const EsqlTab = tabWithSuspense(lazy(() => import('../esql_tab_content'))); +// const AssistantTab = tabWithSuspense(lazy(() => import('../assistant_tab_content'))); interface BasicTimelineTab { renderCellValue: (props: CellValueElementProps) => React.ReactNode; rowRenderers: RowRenderer[]; @@ -101,24 +95,6 @@ interface BasicTimelineTab { timelineDescription: string; } -const AssistantTab: React.FC<{ - shouldRefocusPrompt: boolean; - setConversationId: Dispatch>; -}> = memo(({ shouldRefocusPrompt, setConversationId }) => ( - }> - - - - -)); - -AssistantTab.displayName = 'AssistantTab'; - type ActiveTimelineTabProps = BasicTimelineTab & { activeTimelineTab: TimelineTabs; showTimeline: boolean; @@ -159,12 +135,32 @@ const ActiveTimelineTab = memo( [activeTimelineTab] ); - const conversations = useConversationStore(); - - const hasTimelineConversationStarted = useMemo( - () => conversations[TIMELINE_CONVERSATION_TITLE].messages.length > 0, - [conversations] - ); + const memoAssTab = useCallback(() => { + if (showTimeline) { + const AssistantTab = tabWithSuspense(lazy(() => import('../assistant_tab_content'))); + return ( + + ); + } else { + return null; + } + }, [activeTimelineTab, setConversationId, showTimeline]); /* Future developer -> why are we doing that * It is really expansive to re-render the QueryTab because the drag/drop @@ -220,26 +216,7 @@ const ActiveTimelineTab = memo( > {isGraphOrNotesTabs && getTab(activeTimelineTab)} - {hasAssistantPrivilege && ( - - )} + {hasAssistantPrivilege ? memoAssTab() : null} ); } From ddabe950439242353ae5136c6c87e5db2cf0fcdd Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 20 Jan 2024 13:56:55 -0800 Subject: [PATCH 022/141] revert to use localStorage for the last conversation --- .../use_bulk_actions_conversations.ts | 20 ++++++- .../use_fetch_current_user_conversations.ts | 24 -------- .../impl/assistant/assistant_header/index.tsx | 5 +- .../assistant/assistant_overlay/index.tsx | 25 +++----- .../impl/assistant/assistant_title/index.tsx | 6 +- .../conversation_selector/index.tsx | 8 +-- .../impl/assistant/index.tsx | 60 ++++++++++++------- .../impl/assistant_context/constants.tsx | 1 + .../impl/assistant_context/index.tsx | 32 +++++++--- .../connector_selector_inline.tsx | 10 ++-- 10 files changed, 102 insertions(+), 89 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 2fb46215d3a4b..97e3d0a634f0a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -73,6 +73,9 @@ export const bulkChangeConversations = ( http: HttpSetup, conversationsActions: ConversationsBulkActions ) => { + const conversationIdsToDelete = conversationsActions.delete?.ids.filter( + (cId) => !(conversationsActions.create ?? {})[cId] && !(conversationsActions.update ?? {})[cId] + ); return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { method: 'POST', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, @@ -80,7 +83,10 @@ export const bulkChangeConversations = ( update: conversationsActions.update ? Object.keys(conversationsActions.update).reduce( (conversationsToUpdate: ConversationUpdateParams[], conversationId) => { - if (conversationsActions.update) { + if ( + conversationsActions.update && + !conversationsActions.delete?.ids.includes(conversationId) + ) { conversationsToUpdate.push({ id: conversationId, ...conversationsActions.update[conversationId], @@ -94,7 +100,10 @@ export const bulkChangeConversations = ( create: conversationsActions.create ? Object.keys(conversationsActions.create).reduce( (conversationsToCreate: Conversation[], conversationId: string) => { - if (conversationsActions.create) { + if ( + conversationsActions.create && + !conversationsActions.delete?.ids.includes(conversationId) + ) { conversationsToCreate.push(conversationsActions.create[conversationId]); } return conversationsToCreate; @@ -102,7 +111,12 @@ export const bulkChangeConversations = ( [] ) : undefined, - delete: conversationsActions.delete, + delete: + conversationIdsToDelete && conversationIdsToDelete.length > 0 + ? { + ids: conversationIdsToDelete, + } + : undefined, }), }); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts index 08f3e65170f1e..6814562c06a86 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts @@ -10,7 +10,6 @@ import { useKibana } from '@kbn/kibana-react-plugin/public'; import { useQuery } from '@tanstack/react-query'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, - ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; import { Conversation } from '../../../assistant_context/types'; @@ -51,26 +50,3 @@ export const useFetchCurrentUserConversations = ( return querySt; }; - -/** - * API call for getting conversation by id. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} options.id - Conversation id. - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const useLastConversation = () => { - const { http } = useKibana().services; - - const querySt = useQuery([ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST], () => - http.fetch(ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, { - method: 'GET', - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - }) - ); - - return { ...querySt }; -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index b1f35b47701b7..625091f6b5b77 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -88,7 +88,10 @@ export const AssistantHeader: React.FC = ({ isDisabled={isDisabled} docLinks={docLinks} selectedConversation={currentConversation} - setCurrentConversation={setCurrentConversation} + onChange={(updatedConversation) => { + setCurrentConversation(updatedConversation); + onConversationSelected(updatedConversation.id); + }} title={title} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx index 62100d29015d0..44ae5bbedad08 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { EuiModal } from '@elastic/eui'; import useEvent from 'react-use/lib/useEvent'; @@ -14,7 +14,6 @@ import styled from 'styled-components'; import { ShowAssistantOverlayProps, useAssistantContext } from '../../assistant_context'; import { Assistant } from '..'; import { WELCOME_CONVERSATION_TITLE } from '../use_conversation/translations'; -import { useLastConversation } from '../api/conversations/use_fetch_current_user_conversations'; const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; @@ -34,16 +33,8 @@ export const AssistantOverlay = React.memo(() => { WELCOME_CONVERSATION_TITLE ); const [promptContextId, setPromptContextId] = useState(); - const { assistantTelemetry, setShowAssistantOverlay } = useAssistantContext(); - - const { data: lastConversation, isLoading } = useLastConversation(); - - const lastConversationId = useMemo(() => { - if (!isLoading) { - return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; - } - return WELCOME_CONVERSATION_TITLE; - }, [isLoading, lastConversation?.id]); + const { assistantTelemetry, setShowAssistantOverlay, getLastConversationId } = + useAssistantContext(); // Bind `showAssistantOverlay` in SecurityAssistantContext to this modal instance const showOverlay = useCallback( @@ -53,7 +44,7 @@ export const AssistantOverlay = React.memo(() => { promptContextId: pid, conversationId: cid, }: ShowAssistantOverlayProps) => { - const newConversationId = cid ?? lastConversationId; + const newConversationId = getLastConversationId(cid); if (so) assistantTelemetry?.reportAssistantInvoked({ conversationId: newConversationId, @@ -64,7 +55,7 @@ export const AssistantOverlay = React.memo(() => { setPromptContextId(pid); setConversationId(newConversationId); }, - [assistantTelemetry, lastConversationId] + [assistantTelemetry, getLastConversationId] ); useEffect(() => { setShowAssistantOverlay(showOverlay); @@ -74,15 +65,15 @@ export const AssistantOverlay = React.memo(() => { const handleShortcutPress = useCallback(() => { // Try to restore the last conversation on shortcut pressed if (!isModalVisible) { - setConversationId(lastConversationId); + setConversationId(getLastConversationId()); assistantTelemetry?.reportAssistantInvoked({ invokedBy: 'shortcut', - conversationId: lastConversationId, + conversationId: getLastConversationId(), }); } setIsModalVisible(!isModalVisible); - }, [isModalVisible, lastConversationId, assistantTelemetry]); + }, [isModalVisible, getLastConversationId, assistantTelemetry]); // Register keyboard listener to show the modal when cmd + ; is pressed const onKeyDown = useCallback( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx index 45602d6b08180..d4dee2453fd0e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx @@ -32,8 +32,8 @@ export const AssistantTitle: React.FC<{ title: string | JSX.Element; docLinks: Omit; selectedConversation: Conversation | undefined; - setCurrentConversation: React.Dispatch>; -}> = ({ isDisabled = false, title, docLinks, selectedConversation, setCurrentConversation }) => { + onChange: (updatedConversation: Conversation) => void; +}> = ({ isDisabled = false, title, docLinks, selectedConversation, onChange }) => { const selectedConnectorId = selectedConversation?.apiConfig?.connectorId; const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks; @@ -113,7 +113,7 @@ export const AssistantTitle: React.FC<{ isDisabled={isDisabled || selectedConversation === undefined} selectedConnectorId={selectedConnectorId} selectedConversation={selectedConversation} - setCurrentConversation={setCurrentConversation} + onConnectorSelected={onChange} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 519192f756406..e193fbd62a9d3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -128,13 +128,11 @@ export const ConversationSelector: React.FC = React.memo( // Callback for when user deletes a conversation const onDelete = useCallback( (cId: string) => { + onConversationDeleted(cId); if (selectedConversationId === cId) { - onConversationSelected( - getPreviousConversationId(conversationIds, cId), - conversations[cId].title - ); + const prevConversationId = getPreviousConversationId(conversationIds, cId); + onConversationSelected(prevConversationId, conversations[prevConversationId].title); } - onConversationDeleted(cId); }, [ selectedConversationId, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index d8ec053a599a5..04ed1c7da78d5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -53,7 +53,6 @@ import { ConnectorMissingCallout } from '../connectorland/connector_missing_call import { FetchConversationsResponse, useFetchCurrentUserConversations, - useLastConversation, } from './api/conversations/use_fetch_current_user_conversations'; import { Conversation } from '../assistant_context/types'; @@ -88,11 +87,16 @@ const AssistantComponent: React.FC = ({ getComments, http, promptContexts, + setLastConversationId, + getLastConversationId, title, allSystemPrompts, baseConversations, } = useAssistantContext(); + const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = + useConversation(); + const [selectedPromptContexts, setSelectedPromptContexts] = useState< Record >({}); @@ -132,27 +136,12 @@ const AssistantComponent: React.FC = ({ refetch, } = useFetchCurrentUserConversations(onFetchedConversations); - const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = - useConversation(); useEffect(() => { if (!isLoading && !isError) { setConversations(conversationsData ?? {}); } }, [conversationsData, isError, isLoading]); - const { - data: lastConversation, - isLoading: isLoadingLast, - refetch: refetchLastUpdated, - } = useLastConversation(); - - const lastConversationId = useMemo(() => { - if (!isLoadingLast) { - return lastConversation?.id ?? WELCOME_CONVERSATION_TITLE; - } - return WELCOME_CONVERSATION_TITLE; - }, [isLoadingLast, lastConversation?.id]); - const refetchResults = useCallback(async () => { const updatedConv = await refetch(); if (!updatedConv.isLoading) { @@ -176,7 +165,7 @@ const AssistantComponent: React.FC = ({ ); const [selectedConversationId, setSelectedConversationId] = useState( - isAssistantEnabled ? lastConversationId : WELCOME_CONVERSATION_TITLE + isAssistantEnabled ? getLastConversationId(conversationId) : WELCOME_CONVERSATION_TITLE ); useEffect(() => { @@ -191,20 +180,30 @@ const AssistantComponent: React.FC = ({ const refetchCurrentConversation = useCallback( async (cId?: string) => { + if ( + (!cId && selectedConversationId === currentConversation.title) || + !conversations[selectedConversationId] + ) { + return; + } const updatedConversation = await getConversation(cId ?? selectedConversationId); if (updatedConversation) { setCurrentConversation(updatedConversation); } return updatedConversation; }, - [getConversation, selectedConversationId] + [conversations, currentConversation.title, getConversation, selectedConversationId] ); useEffect(() => { - if (!isLoadingLast && lastConversation && lastConversation.id) { - setCurrentConversation(lastConversation); + if (!isLoading) { + const conversation = + conversations[selectedConversationId ?? getLastConversationId(conversationId)]; + if (conversation) { + setCurrentConversation(conversation); + } } - }, [isLoadingLast, lastConversation]); + }, [conversationId, conversations, getLastConversationId, isLoading, selectedConversationId]); // Welcome setup state const isWelcomeSetup = useMemo(() => { @@ -227,6 +226,18 @@ const AssistantComponent: React.FC = ({ // Settings modal state (so it isn't shared between assistant instances like Timeline) const [isSettingsModalVisible, setIsSettingsModalVisible] = useState(false); + // Remember last selection for reuse after keyboard shortcut is pressed. + // Clear it if there is no connectors + useEffect(() => { + if (areConnectorsFetched && !connectors?.length) { + return setLastConversationId(WELCOME_CONVERSATION_TITLE); + } + + if (!currentConversation.excludeFromLastConversationStorage) { + setLastConversationId(currentConversation.id); + } + }, [areConnectorsFetched, connectors?.length, currentConversation, setLastConversationId]); + const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ conversation: blockBotConversation, conversations, @@ -314,6 +325,7 @@ const AssistantComponent: React.FC = ({ const refetchedConversation = await refetchCurrentConversation(cId); if (refetchedConversation) { setCurrentConversation(refetchedConversation); + setConversations({ ...(conversations ?? {}), [cId]: refetchedConversation }); } setEditingSystemPromptId( getDefaultSystemPrompt({ allSystemPrompts, conversation: refetchedConversation })?.id @@ -559,8 +571,10 @@ const AssistantComponent: React.FC = ({ conversations={conversations} onConversationDeleted={handleOnConversationDeleted} refetchConversationsState={async () => { - await refetchResults(); - await refetchCurrentConversation(); + const refetchedConversations = await refetchResults(); + if (refetchCurrentConversation[selectedConversationId]) { + await refetchCurrentConversation(); + } }} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx index b435e11256359..cc747a705b851 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx @@ -10,6 +10,7 @@ import { KnowledgeBaseConfig } from '../assistant/types'; export const DEFAULT_ASSISTANT_NAMESPACE = 'elasticAssistantDefault'; export const QUICK_PROMPT_LOCAL_STORAGE_KEY = 'quickPrompts'; export const SYSTEM_PROMPT_LOCAL_STORAGE_KEY = 'systemPrompts'; +export const LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY = 'lastConversationId'; export const KNOWLEDGE_BASE_LOCAL_STORAGE_KEY = 'knowledgeBase'; /** The default `n` latest alerts, ordered by risk score, sent as context to the assistant */ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 30addae96583e..c912cc9472dc9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -31,12 +31,14 @@ import { DEFAULT_ASSISTANT_NAMESPACE, DEFAULT_KNOWLEDGE_BASE_SETTINGS, KNOWLEDGE_BASE_LOCAL_STORAGE_KEY, + LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY, QUICK_PROMPT_LOCAL_STORAGE_KEY, SYSTEM_PROMPT_LOCAL_STORAGE_KEY, } from './constants'; import { CONVERSATIONS_TAB, SettingsTabs } from '../assistant/settings/assistant_settings'; import { AssistantAvailability, AssistantTelemetry } from './types'; import { useCapabilities } from '../assistant/api/capabilities/use_capabilities'; +import { WELCOME_CONVERSATION_TITLE } from '../assistant/use_conversation/translations'; export interface ShowAssistantOverlayProps { showOverlay: boolean; @@ -87,7 +89,6 @@ export interface AssistantProviderProps { http: HttpSetup; baseConversations: Record; nameSpace?: string; - // setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; title?: string; @@ -133,6 +134,7 @@ export interface UseAssistantContext { }) => EuiCommentProps[]; http: HttpSetup; knowledgeBase: KnowledgeBaseConfig; + getLastConversationId: (id?: string) => string; promptContexts: Record; modelEvaluatorEnabled: boolean; nameSpace: string; @@ -140,10 +142,10 @@ export interface UseAssistantContext { selectedSettingsTab: SettingsTabs; setAllQuickPrompts: React.Dispatch>; setAllSystemPrompts: React.Dispatch>; - // setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; setKnowledgeBase: React.Dispatch>; + setLastConversationId: React.Dispatch>; setSelectedSettingsTab: React.Dispatch>; setShowAssistantOverlay: (showAssistantOverlay: ShowAssistantOverlay) => void; showAssistantOverlay: ShowAssistantOverlay; @@ -195,6 +197,9 @@ export const AssistantProvider: React.FC = ({ baseSystemPrompts ); + const [localStorageLastConversationId, setLocalStorageLastConversationId] = + useLocalStorage(`${nameSpace}.${LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY}`); + /** * Local storage for knowledge base configuration, prefixed by assistant nameSpace */ @@ -248,6 +253,14 @@ export const AssistantProvider: React.FC = ({ */ const [selectedSettingsTab, setSelectedSettingsTab] = useState(CONVERSATIONS_TAB); + const getLastConversationId = useCallback( + // if a conversationId has been provided, use that + // if not, check local storage + // last resort, go to welcome conversation + (id?: string) => id ?? localStorageLastConversationId ?? WELCOME_CONVERSATION_TITLE, + [localStorageLastConversationId] + ); + // Fetch assistant capabilities const { data: capabilities } = useCapabilities({ http, toasts }); const { assistantModelEvaluation: modelEvaluatorEnabled, assistantStreamingEnabled } = @@ -282,7 +295,6 @@ export const AssistantProvider: React.FC = ({ selectedSettingsTab, setAllQuickPrompts: setLocalStorageQuickPrompts, setAllSystemPrompts: setLocalStorageSystemPrompts, - // setConversations: onConversationsUpdated, setDefaultAllow, setDefaultAllowReplacement, setKnowledgeBase: setLocalStorageKnowledgeBase, @@ -292,6 +304,8 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, + getLastConversationId, + setLastConversationId: setLocalStorageLastConversationId, baseConversations, }), [ @@ -301,6 +315,8 @@ export const AssistantProvider: React.FC = ({ assistantStreamingEnabled, assistantTelemetry, augmentMessageCodeBlocks, + localStorageQuickPrompts, + localStorageSystemPrompts, baseAllow, baseAllowReplacement, basePath, @@ -313,22 +329,22 @@ export const AssistantProvider: React.FC = ({ getComments, http, localStorageKnowledgeBase, - localStorageQuickPrompts, - localStorageSystemPrompts, modelEvaluatorEnabled, - nameSpace, promptContexts, + nameSpace, registerPromptContext, selectedSettingsTab, + setLocalStorageQuickPrompts, + setLocalStorageSystemPrompts, setDefaultAllow, setDefaultAllowReplacement, setLocalStorageKnowledgeBase, - setLocalStorageQuickPrompts, - setLocalStorageSystemPrompts, showAssistantOverlay, title, toasts, unRegisterPromptContext, + getLastConversationId, + setLocalStorageLastConversationId, baseConversations, ] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx index 4f9069b5d7840..520240cfb1581 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx @@ -23,7 +23,7 @@ interface Props { isDisabled?: boolean; selectedConnectorId?: string; selectedConversation?: Conversation; - setCurrentConversation: React.Dispatch>; + onConnectorSelected: (conversation: Conversation) => void; } const inputContainerClassName = css` @@ -66,7 +66,7 @@ const placeholderButtonClassName = css` * A compact wrapper of the ConnectorSelector component used in the Settings modal. */ export const ConnectorSelectorInline: React.FC = React.memo( - ({ isDisabled = false, selectedConnectorId, selectedConversation, setCurrentConversation }) => { + ({ isDisabled = false, selectedConnectorId, selectedConversation, onConnectorSelected }) => { const [isOpen, setIsOpen] = useState(false); const { actionTypeRegistry, assistantAvailability, http } = useAssistantContext(); const { setApiConfig } = useConversation(); @@ -106,7 +106,7 @@ export const ConnectorSelectorInline: React.FC = React.memo( setIsOpen(false); if (selectedConversation != null) { - const res = await setApiConfig({ + const conversation = await setApiConfig({ conversationId: selectedConversation.id, title: selectedConversation.title, isDefault: selectedConversation.isDefault, @@ -119,10 +119,10 @@ export const ConnectorSelectorInline: React.FC = React.memo( model: model ?? config?.defaultModel, }, }); - setCurrentConversation(res as Conversation); + onConnectorSelected(conversation as Conversation); } }, - [selectedConversation, setApiConfig, setCurrentConversation] + [selectedConversation, setApiConfig, onConnectorSelected] ); return ( From a455949412f10d462eb23a01c9db4dcb1e474aa7 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 20 Jan 2024 14:20:26 -0800 Subject: [PATCH 023/141] - --- x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 04ed1c7da78d5..776dbad58bf23 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -572,7 +572,7 @@ const AssistantComponent: React.FC = ({ onConversationDeleted={handleOnConversationDeleted} refetchConversationsState={async () => { const refetchedConversations = await refetchResults(); - if (refetchCurrentConversation[selectedConversationId]) { + if (refetchedConversations && refetchedConversations[selectedConversationId]) { await refetchCurrentConversation(); } }} From 0f071b8631a9a8610bdfd853b62f0d8329caff00 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 20 Jan 2024 17:14:02 -0800 Subject: [PATCH 024/141] added appendMessages API --- .../kbn-elastic-assistant-common/constants.ts | 6 +- .../append_conversation_messages.ts | 100 ++++++++++++++++++ .../get_last_conversation.ts | 43 -------- .../server/conversations_data_client/index.ts | 15 ++- ... => append_conversation_messages_route.ts} | 39 +++++-- .../routes/conversation/create_route.ts | 9 +- .../routes/conversation/delete_route.ts | 15 ++- .../server/routes/conversation/find_route.ts | 10 +- .../find_user_conversations_route.ts | 6 -- .../server/routes/conversation/read_route.ts | 11 +- .../routes/conversation/update_route.ts | 26 ++--- .../server/routes/register_routes.ts | 4 +- ...ost_actions_connector_execute_route.gen.ts | 4 + ...ulk_crud_anonimization_fields_route.gen.ts | 4 + .../find_prompts_route.gen.ts | 4 + .../bulk_crud_conversations_route.gen.ts | 4 + .../conversations/common_attributes.gen.ts | 12 +++ .../common_attributes.schema.yaml | 12 +++ .../crud_conversation_route.gen.ts | 29 +++++ .../crud_conversation_route.schema.yaml | 42 ++++++++ .../find_conversations_route.gen.ts | 4 + .../evaluate/post_evaluate_route.gen.ts | 4 + .../knowledge_base/crud_kb_route.gen.ts | 10 +- .../knowledge_base/crud_kb_route.schema.yaml | 3 - .../schemas/prompts/crud_prompts_route.gen.ts | 4 + .../schemas/prompts/find_prompts_route.gen.ts | 4 + 26 files changed, 306 insertions(+), 118 deletions(-) create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts rename x-pack/plugins/elastic_assistant/server/routes/conversation/{read_last_route.ts => append_conversation_messages_route.ts} (51%) diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index 959852fb9c4a7..ae6f951af72f0 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -12,15 +12,15 @@ export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/current_user`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{conversationId}`; -export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_last`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{id}`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID}/messages`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/prompts`; -export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{promptId}`; +export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{id}`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL = `${ELASTIC_AI_ASSISTANT_URL}/anonimization_fields`; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts new file mode 100644 index 0000000000000..5b177a0a7677c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { + ConversationResponse, + Message, + UUID, +} from '../schemas/conversations/common_attributes.gen'; +import { getConversation } from './get_conversation'; + +export const appendConversationMessages = async ( + esClient: ElasticsearchClient, + logger: Logger, + conversationIndex: string, + userId: UUID, + existingConversation: ConversationResponse, + messages: Message[] +): Promise => { + const updatedAt = new Date().toISOString(); + + const params = transformToUpdateScheme(updatedAt, [ + ...(existingConversation.messages ?? []), + ...messages, + ]); + try { + const response = await esClient.updateByQuery({ + conflicts: 'proceed', + index: conversationIndex, + query: { + ids: { + values: [existingConversation.id ?? ''], + }, + }, + refresh: false, + script: { + lang: 'painless', + params: { + ...params, + }, + source: ` + if (params.assignEmpty == true || params.containsKey('messages')) { + def messages = []; + for (message in params.messages) { + def newMessage = [:]; + newMessage['@timestamp'] = message['@timestamp']; + newMessage.content = message.content; + newMessage.is_error = message.is_error; + newMessage.presentation = message.presentation; + newMessage.reader = message.reader; + newMessage.replacements = message.replacements; + newMessage.role = message.role; + messages.add(newMessage); + } + ctx._source.messages = messages; + } + ctx._source.updated_at = params.updated_at; + `, + }, + }); + if (response.failures && response.failures.length > 0) { + logger.warn( + `Error appending conversation messages: ${response.failures.map( + (f) => f.id + )} for conversation by ID: ${existingConversation.id}` + ); + return null; + } + } catch (err) { + logger.warn( + `Error appending conversation messages: ${err} for conversation by ID: ${existingConversation.id}` + ); + throw err; + } + return getConversation(esClient, conversationIndex, existingConversation.id ?? ''); +}; + +export const transformToUpdateScheme = (updatedAt: string, messages: Message[]) => { + return { + updated_at: updatedAt, + messages: messages?.map((message) => ({ + '@timestamp': message.timestamp, + content: message.content, + is_error: message.isError, + presentation: message.presentation, + reader: message.reader, + replacements: message.replacements, + role: message.role, + trace_data: { + trace_id: message.traceData?.traceId, + transaction_id: message.traceData?.transactionId, + }, + })), + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts deleted file mode 100644 index 35ba89d919a62..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_last_conversation.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from '@kbn/core/server'; - -import { ConversationResponse, UUID } from '../schemas/conversations/common_attributes.gen'; -import { SearchEsConversationSchema } from './types'; -import { transformESToConversations } from './transforms'; - -export const getLastConversation = async ( - esClient: ElasticsearchClient, - conversationIndex: string, - userId: UUID -): Promise => { - const response = await esClient.search({ - body: { - sort: { - updated_at: { - order: 'desc', - }, - }, - query: { - bool: { - filter: [{ term: { 'user.id': userId } }], - must_not: { - term: { excludeFromLastConversationStorage: false }, - }, - }, - }, - size: 1, - }, - _source: true, - ignore_unavailable: true, - index: conversationIndex, - seq_no_primary_term: true, - }); - const conversation = transformESToConversations(response); - return conversation[0] ?? null; -}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 54dd0a3d5e352..1480ec5af5999 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -19,13 +19,14 @@ import { ConversationCreateProps, ConversationResponse, ConversationUpdateProps, + Message, } from '../schemas/conversations/common_attributes.gen'; import { FindConversationsResponse } from '../schemas/conversations/find_conversations_route.gen'; import { findConversations } from './find_conversations'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; import { deleteConversation } from './delete_conversation'; -import { getLastConversation } from './get_last_conversation'; +import { appendConversationMessages } from './append_conversation_messages'; export enum OpenAiProviderType { OpenAi = 'OpenAI', @@ -128,13 +129,19 @@ export class AIAssistantConversationsDataClient { * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. * @returns The conversation created */ - public getLastConversation = async (): Promise => { + public appendConversationMessages = async ( + conversation: ConversationResponse, + messages: Message[] + ): Promise => { const { currentUser } = this; const esClient = await this.options.elasticsearchClientPromise; - return getLastConversation( + return appendConversationMessages( esClient, + this.options.logger, this.indexTemplateAndPattern.alias, - currentUser?.profile_uid ?? '' + currentUser?.profile_uid ?? '', + conversation, + messages ); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts similarity index 51% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts index fe9f18b6a18e2..925ea6a580e5c 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_last_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts @@ -9,17 +9,22 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, } from '@kbn/elastic-assistant-common'; import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { + AppendConversationMessageRequestBody, + AppendConversationMessageRequestParams, +} from '../../schemas/conversations/crud_conversation_route.gen'; -export const readLastConversationRoute = (router: ElasticAssistantPluginRouter) => { +export const appendConversationMessageRoute = (router: ElasticAssistantPluginRouter) => { router.versioned - .get({ + .post({ access: 'public', - path: ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_LAST, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, options: { tags: ['access:elasticAssistant'], }, @@ -27,20 +32,36 @@ export const readLastConversationRoute = (router: ElasticAssistantPluginRouter) .addVersion( { version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: false, + validate: { + request: { + body: buildRouteValidationWithZod(AppendConversationMessageRequestBody), + params: buildRouteValidationWithZod(AppendConversationMessageRequestParams), + }, + }, }, async (context, request, response): Promise> => { - const responseObj = buildResponse(response); - + const assistantResponse = buildResponse(response); + const { id } = request.params; + console.log(id) try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); + const existingConversation = await dataClient?.getConversation(id); + if (existingConversation == null) { + return assistantResponse.error({ + body: `conversation id: "${id}" not found`, + statusCode: 404, + }); + } - const conversation = await dataClient?.getLastConversation(); + const conversation = await dataClient?.appendConversationMessages( + existingConversation, + request.body.messages + ); return response.ok({ body: conversation ?? {} }); } catch (err) { const error = transformError(err); - return responseObj.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts index 77fd6555054d4..28d5075022d97 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts @@ -39,12 +39,7 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v }, }, async (context, request, response): Promise> => { - const siemResponse = buildResponse(response); - // const validationErrors = validateCreateRuleProps(request.body); - // if (validationErrors.length) { - // return siemResponse.error({ statusCode: 400, body: validationErrors }); - // } - + const assistantResponse = buildResponse(response); try { const ctx = await context.resolve(['core', 'elasticAssistant']); @@ -55,7 +50,7 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v }); } catch (err) { const error = transformError(err as Error); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts index 7a65d5c3e0972..ac16ec91e7bf7 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts @@ -7,7 +7,6 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, @@ -15,6 +14,8 @@ import { import { ElasticAssistantPluginRouter } from '../../types'; import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; +import { DeleteConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; +import { buildRouteValidationWithZod } from '../route_validation'; export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => { router.versioned @@ -30,28 +31,26 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, validate: { request: { - params: schema.object({ - conversationId: schema.string(), - }), + params: buildRouteValidationWithZod(DeleteConversationRequestParams), }, }, }, async (context, request, response): Promise> => { const assistantResponse = buildResponse(response); try { - const { conversationId } = request.params; + const { id } = request.params; const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const existingConversation = await dataClient?.getConversation(conversationId); + const existingConversation = await dataClient?.getConversation(id); if (existingConversation == null) { return assistantResponse.error({ - body: `conversation id: "${conversationId}" not found`, + body: `conversation id: "${id}" not found`, statusCode: 404, }); } - await dataClient?.deleteConversation(conversationId); + await dataClient?.deleteConversation(id); return response.ok({ body: {} }); } catch (err) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts index 00852a9f8f796..a3757915adc24 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts @@ -39,13 +39,7 @@ export const findConversationsRoute = (router: ElasticAssistantPluginRouter, log }, }, async (context, request, response): Promise> => { - const siemResponse = buildResponse(response); - - /* const validationErrors = validateFindConversationsRequestQuery(request.query); - if (validationErrors.length) { - return siemResponse.error({ statusCode: 400, body: validationErrors }); - }*/ - + const assistantResponse = buildResponse(response); try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); @@ -63,7 +57,7 @@ export const findConversationsRoute = (router: ElasticAssistantPluginRouter, log return response.ok({ body: result }); } catch (err) { const error = transformError(err); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts index 1e8703e4340b8..c8e1353caf4fc 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts @@ -40,12 +40,6 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) }, async (context, request, response): Promise> => { const assistantResponse = buildResponse(response); - - /* const validationErrors = validateFindConversationsRequestQuery(request.query); - if (validationErrors.length) { - return siemResponse.error({ statusCode: 400, body: validationErrors }); - }*/ - try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts index f44f1cf26b56e..143c6036c0bd2 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts @@ -7,7 +7,6 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, @@ -15,6 +14,8 @@ import { import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { ReadConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { router.versioned @@ -30,22 +31,20 @@ export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, validate: { request: { - params: schema.object({ - conversationId: schema.string(), - }), + params: buildRouteValidationWithZod(ReadConversationRequestParams), }, }, }, async (context, request, response): Promise> => { const responseObj = buildResponse(response); - const { conversationId } = request.params; + const { id } = request.params; try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const conversation = await dataClient?.getConversation(conversationId); + const conversation = await dataClient?.getConversation(id); return response.ok({ body: conversation ?? {} }); } catch (err) { const error = transformError(err); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts index 4f758f6feaeea..6cdb50724b4a7 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts @@ -7,7 +7,6 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, @@ -19,6 +18,7 @@ import { ConversationUpdateProps, } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; +import { UpdateConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => { router.versioned @@ -35,28 +35,22 @@ export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => validate: { request: { body: buildRouteValidationWithZod(ConversationUpdateProps), - params: schema.object({ - conversationId: schema.string(), - }), + params: buildRouteValidationWithZod(UpdateConversationRequestParams), }, }, }, async (context, request, response): Promise> => { - const siemResponse = buildResponse(response); - const { conversationId } = request.params; - /* const validationErrors = validateUpdateConversationProps(request.body); - if (validationErrors.length) { - return siemResponse.error({ statusCode: 400, body: validationErrors }); - }*/ + const assistantResponse = buildResponse(response); + const { id } = request.params; try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const existingConversation = await dataClient?.getConversation(conversationId); + const existingConversation = await dataClient?.getConversation(id); if (existingConversation == null) { - return siemResponse.error({ - body: `conversation id: "${conversationId}" not found`, + return assistantResponse.error({ + body: `conversation id: "${id}" not found`, statusCode: 404, }); } @@ -65,8 +59,8 @@ export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => request.body ); if (conversation == null) { - return siemResponse.error({ - body: `conversation id: "${conversationId}" was not updated`, + return assistantResponse.error({ + body: `conversation id: "${id}" was not updated`, statusCode: 400, }); } @@ -75,7 +69,7 @@ export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => }); } catch (err) { const error = transformError(err); - return siemResponse.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index 60ea189b65d10..82a9153fa50ea 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -20,7 +20,7 @@ import { readConversationRoute } from './conversation/read_route'; import { updateConversationRoute } from './conversation/update_route'; import { findUserConversationsRoute } from './conversation/find_user_conversations_route'; import { bulkActionConversationsRoute } from './conversation/bulk_actions_route'; -import { readLastConversationRoute } from './conversation/read_last_route'; +import { appendConversationMessageRoute } from './conversation/append_conversation_messages_route'; import { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; import { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; @@ -42,7 +42,7 @@ export const registerRoutes = ( readConversationRoute(router); updateConversationRoute(router); deleteConversationRoute(router); - readLastConversationRoute(router); + appendConversationMessageRoute(router); // Conversations bulk CRUD bulkActionConversationsRoute(router, logger); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 75d03de1de1a5..85b530c0c153f 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Execute Connector API endpoint + * version: 1 */ export type ExecuteConnectorRequestParams = z.infer; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts index 9cc4c8e8f060f..54418c17908e0 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Bulk Actions API endpoint + * version: 2023-10-31 */ export type BulkActionSkipReason = z.infer; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts index 7b6f638be5d5e..4ffb41d39e359 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts @@ -11,6 +11,10 @@ import { ArrayFromString } from '@kbn/zod-helpers'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Find AnonimizationFields API endpoint + * version: 2023-10-31 */ import { AnonimizationFieldResponse } from './bulk_crud_anonimization_fields_route.gen'; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts index 70dee1301afbc..bb401150bbee0 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Bulk Actions API endpoint + * version: 2023-10-31 */ import { diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts index 48cdb3b73b444..c71442f6048fd 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Common Conversation Attributes + * version: not applicable */ /** @@ -232,3 +236,11 @@ export const ConversationCreateProps = z.object({ excludeFromLastConversationStorage: z.boolean().optional(), replacements: z.array(Replacement).optional(), }); + +export type ConversationMessageCreateProps = z.infer; +export const ConversationMessageCreateProps = z.object({ + /** + * The conversation messages. + */ + messages: z.array(Message), +}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml index ebda501ed1f81..c16054062a6e3 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -230,3 +230,15 @@ components: type: array items: $ref: '#/components/schemas/Replacement' + + ConversationMessageCreateProps: + type: object + required: + - messages + properties: + messages: + type: array + items: + $ref: '#/components/schemas/Message' + description: The conversation messages. + diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts index 18f2ee9ce7491..ed5f1b8f057c6 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Create Conversation API endpoint + * version: 2023-10-31 */ import { @@ -17,8 +21,33 @@ import { ConversationResponse, UUID, ConversationUpdateProps, + ConversationMessageCreateProps, } from './common_attributes.gen'; +export type AppendConversationMessageRequestParams = z.infer< + typeof AppendConversationMessageRequestParams +>; +export const AppendConversationMessageRequestParams = z.object({ + /** + * The conversation's `id` value. + */ + id: UUID, +}); +export type AppendConversationMessageRequestParamsInput = z.input< + typeof AppendConversationMessageRequestParams +>; + +export type AppendConversationMessageRequestBody = z.infer< + typeof AppendConversationMessageRequestBody +>; +export const AppendConversationMessageRequestBody = ConversationMessageCreateProps; +export type AppendConversationMessageRequestBodyInput = z.input< + typeof AppendConversationMessageRequestBody +>; + +export type AppendConversationMessageResponse = z.infer; +export const AppendConversationMessageResponse = ConversationResponse; + export type CreateConversationRequestBody = z.infer; export const CreateConversationRequestBody = ConversationCreateProps; export type CreateConversationRequestBodyInput = z.input; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml index c0b92754102fe..a7f08659e76e3 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml @@ -147,3 +147,45 @@ paths: type: string message: type: string + + /api/elastic_assistant/conversations/{id}/messages: + post: + operationId: AppendConversationMessage + x-codegen-enabled: true + description: Append a message to the conversation + summary: Append a message to the conversation + tags: + - Conversation API + parameters: + - name: id + in: path + required: true + description: The conversation's `id` value. + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/UUID' + requestBody: + required: true + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationMessageCreateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: './common_attributes.schema.yaml#/components/schemas/ConversationResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string \ No newline at end of file diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts index d9c9510655a8f..16743f77b3efd 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts @@ -11,6 +11,10 @@ import { ArrayFromString } from '@kbn/zod-helpers'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Find Conversations API endpoint + * version: 2023-10-31 */ import { ConversationResponse } from './common_attributes.gen'; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts index 6d32a8a24f9b7..737aa132ead74 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Evaluate API endpoint + * version: 2023-10-31 */ export type DatasetItem = z.infer; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts index 1128c0e88a215..634cd8cb6e78b 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: KnowledgeBase API endpoints + * version: 2023-10-31 */ /** @@ -28,7 +32,7 @@ export const CreateKnowledgeBaseRequestParams = z.object({ /** * The KnowledgeBase `resource` value. */ - resource: z.string(), + resource: z.string().optional(), }); export type CreateKnowledgeBaseRequestParamsInput = z.input< typeof CreateKnowledgeBaseRequestParams @@ -42,7 +46,7 @@ export const DeleteKnowledgeBaseRequestParams = z.object({ /** * The KnowledgeBase `resource` value. */ - resource: z.string(), + resource: z.string().optional(), }); export type DeleteKnowledgeBaseRequestParamsInput = z.input< typeof DeleteKnowledgeBaseRequestParams @@ -56,7 +60,7 @@ export const ReadKnowledgeBaseRequestParams = z.object({ /** * The KnowledgeBase `resource` value. */ - resource: z.string(), + resource: z.string().optional(), }); export type ReadKnowledgeBaseRequestParamsInput = z.input; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml index fd0458bf29319..650a7e141ce39 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml @@ -14,7 +14,6 @@ paths: parameters: - name: resource in: path - required: true description: The KnowledgeBase `resource` value. schema: type: string @@ -48,7 +47,6 @@ paths: parameters: - name: resource in: path - required: true description: The KnowledgeBase `resource` value. schema: type: string @@ -89,7 +87,6 @@ paths: parameters: - name: resource in: path - required: true description: The KnowledgeBase `resource` value. schema: type: string diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts index c8c2e8d339945..d2edd0384e598 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts @@ -10,6 +10,10 @@ import { z } from 'zod'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Create Prompt API endpoint + * version: 2023-10-31 */ /** diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts index 4c758384c8a2d..537e9ded995d0 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts @@ -11,6 +11,10 @@ import { ArrayFromString } from '@kbn/zod-helpers'; /* * NOTICE: Do not edit this file manually. * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Find Prompts API endpoint + * version: 2023-10-31 */ import { PromptResponse } from './crud_prompts_route.gen'; From 371876193d0e2cfcc23ddf649285f85d1165883f Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sat, 20 Jan 2024 17:17:11 -0800 Subject: [PATCH 025/141] fixed KB params --- .../routes/conversation/append_conversation_messages_route.ts | 1 - .../server/routes/knowledge_base/delete_knowledge_base.ts | 2 +- .../server/routes/knowledge_base/get_knowledge_base_status.ts | 2 +- .../server/routes/knowledge_base/post_knowledge_base.ts | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts index 925ea6a580e5c..0c6948811f178 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts @@ -42,7 +42,6 @@ export const appendConversationMessageRoute = (router: ElasticAssistantPluginRou async (context, request, response): Promise> => { const assistantResponse = buildResponse(response); const { id } = request.params; - console.log(id) try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts index f6e91cd671332..c3cb647c2996c 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts @@ -42,7 +42,7 @@ export const deleteKnowledgeBaseRoute = ( version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, validate: { request: { - body: buildRouteValidationWithZod(DeleteKnowledgeBaseRequestParams), + params: buildRouteValidationWithZod(DeleteKnowledgeBaseRequestParams), }, }, }, diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts index 924339c698b1a..cffbf3836c1fa 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts @@ -46,7 +46,7 @@ export const getKnowledgeBaseStatusRoute = ( version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, validate: { request: { - body: buildRouteValidationWithZod(ReadKnowledgeBaseRequestParams), + params: buildRouteValidationWithZod(ReadKnowledgeBaseRequestParams), }, }, }, diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts index a9f908117aef8..868cd72c42aef 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts @@ -45,7 +45,7 @@ export const postKnowledgeBaseRoute = ( version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, validate: { request: { - body: buildRouteValidationWithZod(CreateKnowledgeBaseRequestParams), + params: buildRouteValidationWithZod(CreateKnowledgeBaseRequestParams), }, }, }, From e1b161ef505a11027f3433d3ad91324db8b16d0b Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 21 Jan 2024 22:41:04 -0800 Subject: [PATCH 026/141] Added API unit tests --- .../transform_raw_data/index.test.tsx | 16 +- .../api/conversations/conversations.ts | 40 +- .../assistant/chat_send/use_chat_send.tsx | 5 +- .../impl/assistant/use_conversation/index.tsx | 31 +- .../impl/assistant_context/index.tsx | 4 +- .../connectorland/connector_setup/index.tsx | 6 +- .../conversations_data_client.mock.ts | 32 + .../__mocks__/conversations_schema.mock.ts | 84 + .../server/__mocks__/request.ts | 64 + .../server/__mocks__/request_context.ts | 9 +- .../server/__mocks__/response.ts | 34 + ...reate_resource_installation_helper.test.ts | 348 +++ .../server/ai_assistant_service/index.test.ts | 2671 +++++++++++++---- .../append_conversation_messages.ts | 9 +- .../conversations_data_writer.test.ts | 345 +++ .../conversations_data_writer.ts | 3 +- .../create_conversation.test.ts | 79 + .../delete_conversation.test.ts | 99 + .../get_conversation.test.ts | 37 + .../conversations_data_client/index.test.ts | 31 + .../server/conversations_data_client/index.ts | 22 +- .../update_conversation.test.ts | 64 + .../update_conversation.ts | 85 +- .../append_conversation_messages_route.ts | 0 .../conversations/bulk_actions_route.test.ts | 353 +++ .../bulk_actions_route.ts | 0 .../routes/conversations/create_route.test.ts | 190 ++ .../create_route.ts | 0 .../routes/conversations/delete_route.test.ts | 92 + .../delete_route.ts | 0 .../routes/conversations/find_route.test.ts | 97 + .../find_route.ts | 0 .../find_user_conversations_route.test.ts | 97 + .../find_user_conversations_route.ts | 0 .../routes/conversations/read_route.test.ts | 88 + .../read_route.ts | 0 .../routes/conversations/update_route.test.ts | 154 + .../update_route.ts | 0 .../server/routes/register_routes.ts | 16 +- .../public/assistant/get_comments/index.tsx | 12 +- .../assistant/get_comments/stream/index.tsx | 2 +- .../common/mock/mock_assistant_provider.tsx | 4 +- .../rule_status_failed_callout.test.tsx | 5 +- 43 files changed, 4448 insertions(+), 780 deletions(-) create mode 100644 x-pack/plugins/elastic_assistant/server/__mocks__/conversations_data_client.mock.ts create mode 100644 x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts create mode 100644 x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/append_conversation_messages_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/bulk_actions_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/create_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/delete_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/find_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/find_user_conversations_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/read_route.ts (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversation => conversations}/update_route.ts (100%) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx index ded518deece66..1484913c1e37b 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx @@ -9,7 +9,7 @@ import { mockGetAnonymizedValue } from '../../mock/get_anonymized_value'; import { transformRawData } from '.'; describe('transformRawData', () => { - it('returns non-anonymized data when rawData is a string', () => { + it('returns non-anonymized data when rawData is a string', async () => { const inputRawData = { allow: ['field1'], allowReplacement: ['field1', 'field2'], @@ -17,7 +17,7 @@ describe('transformRawData', () => { rawData: 'this will not be anonymized', }; - const result = transformRawData({ + const result = await transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -29,7 +29,7 @@ describe('transformRawData', () => { expect(result).toEqual('this will not be anonymized'); }); - it('calls onNewReplacements with the expected replacements', () => { + it('calls onNewReplacements with the expected replacements', async () => { const inputRawData = { allow: ['field1'], allowReplacement: ['field1'], @@ -39,7 +39,7 @@ describe('transformRawData', () => { const onNewReplacements = jest.fn(); - transformRawData({ + await transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -51,7 +51,7 @@ describe('transformRawData', () => { expect(onNewReplacements).toHaveBeenCalledWith({ '1eulav': 'value1' }); }); - it('returns the expected mix of anonymized and non-anonymized data as a CSV string', () => { + it('returns the expected mix of anonymized and non-anonymized data as a CSV string', async () => { const inputRawData = { allow: ['field1', 'field2'], allowReplacement: ['field1'], // only field 1 will be anonymized @@ -59,7 +59,7 @@ describe('transformRawData', () => { rawData: { field1: ['value1', 'value2'], field2: ['value3', 'value4'] }, }; - const result = transformRawData({ + const result = await transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -71,7 +71,7 @@ describe('transformRawData', () => { expect(result).toEqual('field1,1eulav,2eulav\nfield2,value3,value4'); // only field 1 is anonymized }); - it('omits fields that are not included in the `allow` list, even if they are members of `allowReplacement`', () => { + it('omits fields that are not included in the `allow` list, even if they are members of `allowReplacement`', async () => { const inputRawData = { allow: ['field1', 'field2'], // field3 is NOT allowed allowReplacement: ['field1', 'field3'], // field3 is requested to be anonymized @@ -83,7 +83,7 @@ describe('transformRawData', () => { }, }; - const result = transformRawData({ + const result = await transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts index 60f69dcd8b783..0950b16760456 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts @@ -137,9 +137,12 @@ export interface PutConversationMessageParams { } /** - * API call for evaluating models. + * API call for updating conversation. * * @param {PutConversationMessageParams} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {string} [options.title] - Conversation title + * @param {AbortSignal} [options.signal] - AbortSignal * * @returns {Promise} */ @@ -177,3 +180,38 @@ export const updateConversationApi = async ({ return error as IHttpFetchError; } }; + +/** + * API call for evaluating models. + * + * @param {PutConversationMessageParams} options - The options object. + * + * @returns {Promise} + */ +export const appendConversationMessagesApi = async ({ + http, + conversationId, + messages, + signal, +}: PutConversationMessageParams): Promise => { + try { + const response = await http.fetch( + `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${conversationId}/messages`, + { + method: 'POST', + body: JSON.stringify({ + messages, + }), + headers: { + 'Content-Type': 'application/json', + }, + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + signal, + } + ); + + return response as Conversation; + } catch (error) { + return error as IHttpFetchError; + } +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 8ed592a77c79a..86bda9b22a222 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -103,7 +103,10 @@ export const useChatSend = ({ }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); - await appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + await appendMessage({ + conversationId: currentConversation.id, + message: responseMessage, + }); await refresh(); }, [ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 6ac67fb022899..8395b1fd95428 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -13,6 +13,7 @@ import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; import { getDefaultSystemPrompt } from './helpers'; import { + appendConversationMessagesApi, createConversationApi, deleteConversationApi, getConversationById, @@ -131,10 +132,9 @@ export const useConversation = (): UseConversation => { if (prevConversation != null) { const { messages } = prevConversation; const message = messages[messages.length - 1]; - const updatedMessages = message - ? [...messages.slice(0, -1), { ...message, content }] - : [...messages]; - await updateConversationApi({ + const updatedMessages = message ? [{ ...message, content }] : []; + + await appendConversationMessagesApi({ http, conversationId, messages: updatedMessages, @@ -148,28 +148,23 @@ export const useConversation = (): UseConversation => { * Append a message to the conversation[] for a given conversationId */ const appendMessage = useCallback( - async ({ conversationId, message }: AppendMessageProps): Promise => { + async ({ conversationId, message }: AppendMessageProps): Promise => { assistantTelemetry?.reportAssistantMessageSent({ conversationId, role: message.role, isEnabledKnowledgeBase, isEnabledRAGAlerts, }); - let messages: Message[] = []; - const prevConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(prevConversation)) { - return []; - } - if (prevConversation != null) { - messages = [...prevConversation.messages, message]; - await updateConversationApi({ - http, - conversationId, - messages, - }); + const res = await appendConversationMessagesApi({ + http, + conversationId, + messages: [message], + }); + if (isHttpFetchError(res)) { + return; } - return messages; + return res.messages; }, [assistantTelemetry, isEnabledKnowledgeBase, isEnabledRAGAlerts, http] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index c912cc9472dc9..4837d744447bc 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -80,7 +80,7 @@ export interface AssistantProviderProps { }: { conversationId: string; content: string; - }) => void; + }) => Promise; currentConversation: Conversation; isFetchingResponse: boolean; regenerateMessage: (conversationId: string) => void; @@ -128,7 +128,7 @@ export interface UseAssistantContext { }: { conversationId: string; content: string; - }) => void; + }) => Promise; regenerateMessage: () => void; showAnonymizedValues: boolean; }) => EuiCommentProps[]; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 7bae68ef0bc49..966cc6400c7bd 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -174,13 +174,13 @@ export const useConnectorSetup = ({ ); const onSaveConnector = useCallback( - (connector: ActionConnector) => { + async (connector: ActionConnector) => { const config = getGenAiConfig(connector); // add action type title to new connector const connectorTypeTitle = getActionTypeTitle(actionTypeRegistry.get(connector.actionTypeId)); // persist only the active conversation - setApiConfig({ + await setApiConfig({ conversationId: conversation.id, title: conversation.title, isDefault: conversation.isDefault, @@ -210,7 +210,7 @@ export const useConnectorSetup = ({ refetchConnectors?.(); setIsConnectorModalVisible(false); - appendMessage({ + await appendMessage({ conversationId: conversation.id, message: { role: 'assistant', diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_data_client.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_data_client.mock.ts new file mode 100644 index 0000000000000..990a68c99c9a8 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_data_client.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PublicMethodsOf } from '@kbn/utility-types'; +import { AIAssistantConversationsDataClient } from '../conversations_data_client'; + +type ConversationsDataClientContract = PublicMethodsOf; +export type ConversationsDataClientMock = jest.Mocked; + +const createConversationsDataClientMock = () => { + const mocked: ConversationsDataClientMock = { + findConversations: jest.fn(), + appendConversationMessages: jest.fn(), + createConversation: jest.fn(), + deleteConversation: jest.fn(), + getConversation: jest.fn(), + updateConversation: jest.fn(), + getReader: jest.fn(), + getWriter: jest.fn().mockResolvedValue({ bulk: jest.fn() }), + }; + return mocked; +}; + +export const conversationsDataClientMock: { + create: () => ConversationsDataClientMock; +} = { + create: createConversationsDataClientMock, +}; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts new file mode 100644 index 0000000000000..9944a9efe7b86 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getCreateConversationSchemaMock = (ruleId = 'rule-1'): QueryRuleCreateProps => ({ + description: 'Detecting root and admin users', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'query', + risk_score: 55, + language: 'kuery', + rule_id: ruleId, +}); + +export const getUpdateConversationSchemaMock = (ruleId = 'rule-1'): QueryRuleCreateProps => ({ + description: 'Detecting root and admin users', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'query', + risk_score: 55, + language: 'kuery', + rule_id: ruleId, +}); + +export const getConversationMock = (params: T): SanitizedRule => ({ + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + name: 'Detect Root/Admin Users', + tags: [], + alertTypeId: ruleTypeMappings[params.type], + consumer: 'siem', + params, + createdAt: new Date('2019-12-13T16:40:33.400Z'), + updatedAt: new Date('2019-12-13T16:40:33.400Z'), + schedule: { interval: '5m' }, + enabled: true, + actions: [], + throttle: null, + notifyWhen: null, + createdBy: 'elastic', + updatedBy: 'elastic', + apiKeyOwner: 'elastic', + muteAll: false, + mutedInstanceIds: [], + scheduledTaskId: '2dabe330-0702-11ea-8b50-773b89126888', + executionStatus: { + status: 'unknown', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + }, + revision: 0, +}); + +export const getQueryConversationParams = (): QueryRuleParams => { + return { + ...getBaseRuleParams(), + type: 'query', + language: 'kuery', + query: 'user.name: root or user.name: admin', + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + dataViewId: undefined, + filters: [ + { + query: { + match_phrase: { + 'host.name': 'some-host', + }, + }, + }, + ], + savedId: undefined, + alertSuppression: undefined, + responseActions: undefined, + }; +}; + +export const getPerformBulkActionSchemaMock = (): PerformBulkActionRequestBody => ({ + query: '', + ids: undefined, + action: BulkActionTypeEnum.disable, +}); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 0edb31a0f0b49..9d6f2b977d605 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -10,6 +10,16 @@ import { EvaluateRequestBodyInput, EvaluateRequestQueryInput, } from '../schemas/evaluate/post_evaluate_route.gen'; +import { + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, +} from '@kbn/elastic-assistant-common'; +import { + getCreateConversationSchemaMock, + getUpdateConversationSchemaMock, +} from './conversations_schema.mock'; export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -55,3 +65,57 @@ export const getPostEvaluateRequest = ({ path: EVALUATE, query, }); + +export const getFindRequest = () => + requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + }); + +export const getDeleteConversationRequest = () => + requestMock.create({ + method: 'delete', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + query: { id: 'conversation-1' }, + }); + +export const getCreateConversationRequest = () => + requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: getCreateConversationSchemaMock(), + }); + +export const getUpdateConversationRequest = () => + requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: getUpdateConversationSchemaMock(), + }); + +export const getConversationReadRequest = () => + requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + query: { id: 'conversation-1' }, + }); + +export const getConversationReadRequestWithId = (id: string) => + requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + query: { id }, + }); + +export const getConversationsBulkActionRequest = () => + requestMock.create({ + method: 'patch', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + body: { + create: [], + update: [], + delete: { + ids: [], + }, + }, + }); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index 7bb899b289d84..2dcf83d34cc4c 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -14,6 +14,7 @@ import { ElasticAssistantRequestHandlerContext, } from '../types'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; +import { conversationsDataClientMock } from './conversations_data_client.mock'; export const createMockClients = () => { const core = coreMock.createRequestHandlerContext(); @@ -28,6 +29,9 @@ export const createMockClients = () => { getRegisteredTools: jest.fn(), logger: loggingSystemMock.createLogger(), telemetry: coreMock.createSetup().analytics, + getAIAssistantConversationsDataClient: conversationsDataClientMock.create(), + getAIAssistantPromptsSOClient: jest.fn(), + getAIAssistantAnonimizationFieldsSOClient: jest.fn(), }, savedObjectsClient: core.savedObjects.client, @@ -78,8 +82,9 @@ const createElasticAssistantRequestContextMock = ( getRegisteredFeatures: jest.fn(), getRegisteredTools: jest.fn(), logger: clients.elasticAssistant.logger, - getAIAssistantDataClient: jest.fn(), - getAIAssistantSOClient: jest.fn(), + getAIAssistantConversationsDataClient: jest.fn(), + getAIAssistantPromptsSOClient: jest.fn(), + getAIAssistantAnonimizationFieldsSOClient: jest.fn(), getCurrentUser: jest.fn(), getServerBasePath: jest.fn(), getSpaceId: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts index 8efe2407f2245..16f3e652e5b0b 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts @@ -6,7 +6,41 @@ */ import { httpServerMock } from '@kbn/core/server/mocks'; +import { getConversationMock, getQueryConversationParams } from './conversations_schema.mock'; +import { estypes } from '@elastic/elasticsearch'; export const responseMock = { create: httpServerMock.createResponseFactory, }; + +export interface FindHit { + page: number; + perPage: number; + total: number; + data: T[]; +} + +export const getEmptyFindResult = (): FindHit => ({ + page: 1, + perPage: 1, + total: 0, + data: [], +}); + +export const getFindConversationsResultWithSingleHit = (): FindHit => ({ + page: 1, + perPage: 1, + total: 1, + data: [getConversationMock(getQueryConversationParams())], +}); + +export const getBasicEmptySearchResponse = (): estypes.SearchResponse => ({ + took: 1, + timed_out: false, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + hits: { + hits: [], + total: { relation: 'eq', value: 0 }, + max_score: 0, + }, +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts new file mode 100644 index 0000000000000..4ac6d4cc69810 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts @@ -0,0 +1,348 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { range } from 'lodash'; +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { + createResourceInstallationHelper, + errorResult, + InitializationPromise, + ResourceInstallationHelper, + successResult, + calculateDelay, + getShouldRetry, +} from './create_resource_installation_helper'; +import { retryUntil } from './test_utils'; + +const logger: ReturnType = + loggingSystemMock.createLogger(); + +const initFn = async (context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { + logger.info(`${context.context}_${namespace}`); +}; + +const initFnWithError = async (context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { + throw new Error('no go'); +}; + +const getCommonInitPromise = async ( + resolution: boolean, + timeoutMs: number = 1, + customLogString: string = '' +): Promise => { + if (timeoutMs < 0) { + throw new Error('fail'); + } + // delay resolution of promise by timeout value + await new Promise((r) => setTimeout(r, timeoutMs)); + const customLog = customLogString && customLogString.length > 0 ? ` - ${customLogString}` : ''; + logger.info(`commonInitPromise resolved${customLog}`); + return Promise.resolve(resolution ? successResult() : errorResult(`error initializing`)); +}; + +const getContextInitialized = async ( + helper: ResourceInstallationHelper, + context: string = 'test1', + namespace: string = DEFAULT_NAMESPACE_STRING +) => { + const { result } = await helper.getInitializedResources(context, namespace); + return result; +}; + +describe('createResourceInstallationHelper', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test(`should wait for commonInitFunction to resolve before calling initFns for registered contexts`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFn + ); + + // Add two contexts that need to be initialized + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + helper.add({ + context: 'test2', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('init fns run', async () => logger.info.mock.calls.length === 3); + + expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); + expect(logger.info).toHaveBeenNthCalledWith(2, 'test1_default'); + expect(logger.info).toHaveBeenNthCalledWith(3, 'test2_default'); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: true, + }); + expect(await helper.getInitializedResources('test2', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: true, + }); + }); + + test(`should return false if context is unrecognized`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFn + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('init fns run', async () => logger.info.mock.calls.length === 2); + + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: true, + }); + expect(await helper.getInitializedResources('test2', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `Unrecognized context test2_default`, + }); + }); + + test(`should log and return false if common init function returns false`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(false, 100), + initFn + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); + + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize context for test1` + ); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `error initializing`, + }); + }); + + test(`should log and return false if common init function throws error`, async () => { + const helper = createResourceInstallationHelper(logger, getCommonInitPromise(true, -1), initFn); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil( + 'common init fns run', + async () => (await getContextInitialized(helper)) === false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - fail`); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `fail`, + }); + }); + + test(`should log and return false if context init function throws error`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFnWithError + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil( + 'context init fns run', + async () => (await getContextInitialized(helper)) === false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - no go`); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `no go`, + }); + }); + + test(`should retry using new common init function if specified`, async () => { + const context = { + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }; + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(false, 100), + initFn + ); + + helper.add(context); + + await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); + + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize context for test1` + ); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `error initializing`, + }); + + helper.retry(context, undefined, getCommonInitPromise(true, 100, 'after retry')); + + await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 2); + expect(logger.info).toHaveBeenCalledWith(`commonInitPromise resolved - after retry`); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: true, + }); + }); + + test(`should retry context init function`, async () => { + const initFnErrorOnce = jest + .fn() + .mockImplementationOnce(() => { + throw new Error('first error'); + }) + .mockImplementation((context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { + logger.info(`${context.context}_${namespace} successfully retried`); + }); + const context = { + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }; + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFnErrorOnce + ); + + helper.add(context); + + await retryUntil( + 'context init fns run', + async () => (await getContextInitialized(helper)) === false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - first error`); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `first error`, + }); + + helper.retry(context, undefined); + + await retryUntil('init fns retried', async () => logger.info.mock.calls.length === 3); + + expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); + expect(logger.info).toHaveBeenNthCalledWith( + 2, + `Retrying resource initialization for context "test1"` + ); + expect(logger.info).toHaveBeenNthCalledWith(3, 'test1_default successfully retried'); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: true, + }); + }); + + test(`should throttle retry`, async () => { + const initFnErrorOnce = jest + .fn() + .mockImplementationOnce(() => { + throw new Error('first error'); + }) + .mockImplementationOnce(() => { + throw new Error('second error'); + }) + .mockImplementation((context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { + logger.info(`${context.context}_${namespace} successfully retried`); + }); + const context = { + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }; + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFnErrorOnce + ); + + helper.add(context); + + await retryUntil( + 'context init fns run', + async () => (await getContextInitialized(helper)) === false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - first error`); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: `first error`, + }); + + logger.info.mockClear(); + logger.error.mockClear(); + + helper.retry(context, undefined); + await new Promise((r) => setTimeout(r, 10)); + helper.retry(context, undefined); + + await retryUntil('init fns retried', async () => { + return logger.error.mock.calls.length === 1; + }); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - second error`); + + // the second retry is throttled so this is never called + expect(logger.info).not.toHaveBeenCalledWith('test1_default successfully retried'); + expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + result: false, + error: 'second error', + }); + }); +}); + +describe('calculateDelay', () => { + test('should return 30 seconds if attempts = 1', () => { + expect(calculateDelay(1)).toEqual(30000); + }); + + test('should return multiple of 5 minutes if attempts > 1', () => { + range(2, 20).forEach((attempt: number) => { + expect(calculateDelay(attempt)).toEqual(Math.pow(2, attempt - 2) * 120000); + }); + }); +}); + +describe('getShouldRetry', () => { + test('should return true if current time is past the previous retry time + the retry delay', () => { + const now = new Date(); + const retry = { + time: new Date(now.setMinutes(now.getMinutes() - 1)).toISOString(), + attempts: 1, + }; + expect(getShouldRetry(retry)).toEqual(true); + }); + + test('should return false if current time is not past the previous retry time + the retry delay', () => { + const now = new Date(); + const retry = { + time: new Date(now.setMinutes(now.getMinutes() - 1)).toISOString(), + attempts: 2, + }; + expect(getShouldRetry(retry)).toEqual(false); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index acbb1b1e60029..b31b70ccb6950 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -5,838 +5,2175 @@ * 2.0. */ -import { - createOrUpdateComponentTemplate, - createOrUpdateIndexTemplate, -} from '@kbn/alerting-plugin/server'; -import { - loggingSystemMock, - elasticsearchServiceMock, - savedObjectsClientMock, -} from '@kbn/core/server/mocks'; -import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; -import type { SavedObject } from '@kbn/core/server'; - -const getSavedObjectConfiguration = (attributes = {}) => ({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'risk-engine-configuration', - id: 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', - namespaces: ['default'], - attributes: { - enabled: false, - ...attributes, +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { ReplaySubject, Subject } from 'rxjs'; +import { IRuleTypeAlerts, RecoveredActionGroup } from '../types'; +import { retryUntil } from './test_utils'; +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { UntypedNormalizedRuleType } from '../rule_type_registry'; +import { getDataStreamAdapter } from './lib/data_stream_adapter'; +import { conversationsDataClientMock } from '../__mocks__/conversations_data_client.mock'; +import { AIAssistantConversationsDataClient } from '../conversations_data_client'; +import { AIAssistantService } from '.'; + +jest.mock('../conversations_data_client'); + +let logger: ReturnType; +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const SimulateTemplateResponse = { + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, }, - references: [], - managed: false, - updated_at: '2023-07-28T09:52:28.768Z', - created_at: '2023-07-28T09:12:26.083Z', - version: 'WzE4MzIsMV0=', - coreMigrationVersion: '8.8.0', - score: 0, }, - ], -}); + mappings: { enabled: false }, + settings: {}, + }, +}; +interface HTTPError extends Error { + statusCode: number; +} + +interface EsError extends Error { + meta: { + body: { + error: { + type: string; + }; + }; + }; +} + +const GetAliasResponse = { + '.internal.alerts-test.alerts-default-000001': { + aliases: { + alias_1: { + is_hidden: true, + }, + alias_2: { + is_hidden: true, + }, + }, + }, +}; -const transformsMock = { - count: 1, - transforms: [ +const GetDataStreamResponse: IndicesGetDataStreamResponse = { + data_streams: [ { - id: 'ml_hostriskscore_pivot_transform_default', - dest: { index: '' }, - source: { index: '' }, + name: 'ignored', + generation: 1, + timestamp_field: { name: 'ignored' }, + hidden: true, + indices: [{ index_name: 'ignored', index_uuid: 'ignored' }], + status: 'green', + template: 'ignored', }, ], }; -jest.mock('@kbn/alerting-plugin/server', () => ({ - createOrUpdateComponentTemplate: jest.fn(), - createOrUpdateIndexTemplate: jest.fn(), -})); +const IlmPutBody = { + policy: { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }, + name: '.alerts-ilm-policy', +}; + +interface GetIndexTemplatePutBodyOpts { + context?: string; + namespace?: string; + useLegacyAlerts?: boolean; + useEcs?: boolean; + secondaryAlias?: string; + useDataStream?: boolean; +} +const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { + const context = opts ? opts.context : undefined; + const namespace = (opts ? opts.namespace : undefined) ?? DEFAULT_NAMESPACE_STRING; + const useLegacyAlerts = opts ? opts.useLegacyAlerts : undefined; + const useEcs = opts ? opts.useEcs : undefined; + const secondaryAlias = opts ? opts.secondaryAlias : undefined; + const useDataStream = opts?.useDataStream ?? false; + + const indexPatterns = useDataStream + ? [`.alerts-${context ? context : 'test'}.alerts-${namespace}`] + : [`.internal.alerts-${context ? context : 'test'}.alerts-${namespace}-*`]; + return { + name: `.alerts-${context ? context : 'test'}.alerts-${namespace}-index-template`, + body: { + index_patterns: indexPatterns, + composed_of: [ + ...(useEcs ? ['.alerts-ecs-mappings'] : []), + `.alerts-${context ? `${context}.alerts` : 'test.alerts'}-mappings`, + ...(useLegacyAlerts ? ['.alerts-legacy-alert-mappings'] : []), + '.alerts-framework-mappings', + ], + ...(useDataStream ? { data_stream: { hidden: true } } : {}), + priority: namespace.length, + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + ...(useDataStream + ? {} + : { + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`, + }, + }), + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': 2500, + }, + mappings: { + dynamic: false, + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace, + }, + }, + ...(secondaryAlias + ? { + aliases: { + [`${secondaryAlias}-default`]: { + is_write_index: false, + }, + }, + } + : {}), + }, + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace, + }, + }, + }; +}; + +const TestRegistrationContext: IRuleTypeAlerts = { + context: 'test', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + shouldWrite: true, +}; -jest.mock('./utils/create_datastream', () => ({ - createDataStream: jest.fn(), -})); +const getContextInitialized = async ( + assistantService: AIAssistantService, + context: string = TestRegistrationContext.context, + namespace: string = DEFAULT_NAMESPACE_STRING +) => { + const { result } = await assistantService.getSpaceResourcesInitializationPromise(namespace); + return result; +}; -jest.mock('../../risk_score/transform/helpers/transforms', () => ({ - createAndStartTransform: jest.fn(), -})); +const conversationsDataClient = conversationsDataClientMock.create(); +const ruleType: jest.Mocked = { + id: 'test.rule-type', + name: 'My test rule', + actionGroups: [{ id: 'default', name: 'Default' }, RecoveredActionGroup], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + recoveryActionGroup: RecoveredActionGroup, + executor: jest.fn(), + category: 'test', + producer: 'alerts', + cancelAlertsOnRuleTimeout: true, + ruleTaskTimeout: '5m', + autoRecoverAlerts: true, + validate: { + params: { validate: (params) => params }, + }, + validLegacyConsumers: [], +}; -jest.mock('./utils/create_index', () => ({ - createIndex: jest.fn(), -})); +const ruleTypeWithAlertDefinition: jest.Mocked = { + ...ruleType, + alerts: TestRegistrationContext as IRuleTypeAlerts<{}>, +}; -jest.spyOn(transforms, 'createTransform').mockResolvedValue(Promise.resolve()); -jest.spyOn(transforms, 'startTransform').mockResolvedValue(Promise.resolve()); +describe('AI Assistant Service', () => { + let pluginStop$: Subject; + + beforeEach(() => { + jest.resetAllMocks(); + logger = loggingSystemMock.createLogger(); + pluginStop$ = new ReplaySubject(1); + jest.spyOn(global.Math, 'random').mockReturnValue(0.01); + clusterClient.indices.simulateTemplate.mockImplementation(async () => SimulateTemplateResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + clusterClient.indices.getDataStream.mockImplementation(async () => GetDataStreamResponse); + }); + + afterEach(() => { + pluginStop$.next(); + pluginStop$.complete(); + }); -describe('RiskEngineDataClient', () => { for (const useDataStreamForAlerts of [false, true]) { const label = useDataStreamForAlerts ? 'data streams' : 'aliases'; + const dataStreamAdapter = getDataStreamAdapter({ useDataStreamForAlerts }); describe(`using ${label} for alert indices`, () => { - let riskEngineDataClient: RiskEngineDataClient; - let mockSavedObjectClient: ReturnType; - let logger: ReturnType; - const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; - const totalFieldsLimit = 1000; - - beforeEach(() => { - logger = loggingSystemMock.createLogger(); - mockSavedObjectClient = savedObjectsClientMock.create(); - const options = { - logger, - kibanaVersion: '8.9.0', - esClient, - soClient: mockSavedObjectClient, - namespace: 'default', - }; - riskEngineDataClient = new RiskEngineDataClient(options); - }); + describe('AIAssistantService()', () => { + test('should correctly initialize common resources', async () => { + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); - afterEach(() => { - jest.clearAllMocks(); - }); + await retryUntil( + 'alert service initialized', + async () => (await assistantService.isInitialized()) === true + ); - describe('getWriter', () => { - it('should return a writer object', async () => { - const writer = await riskEngineDataClient.getWriter({ namespace: 'default' }); - expect(writer).toBeDefined(); - expect(typeof writer?.bulk).toBe('function'); + expect(assistantService.isInitialized()).toEqual(true); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + if (!useDataStreamForAlerts) { + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + } + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); }); - it('should cache and return the same writer for the same namespace', async () => { - const writer1 = await riskEngineDataClient.getWriter({ namespace: 'default' }); - const writer2 = await riskEngineDataClient.getWriter({ namespace: 'default' }); - const writer3 = await riskEngineDataClient.getWriter({ namespace: 'space-1' }); + test('should log error and set initialized to false if adding ILM policy throws error', async () => { + if (useDataStreamForAlerts) return; + + clusterClient.ilm.putLifecycle.mockRejectedValueOnce(new Error('fail')); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); - expect(writer1).toEqual(writer2); - expect(writer2).not.toEqual(writer3); + expect(logger.error).toHaveBeenCalledWith( + `Error installing ILM policy .alerts-ilm-policy - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(1); }); - }); - describe('initializeResources success', () => { - it('should initialize risk engine resources', async () => { - await riskEngineDataClient.initializeResources({ namespace: 'default' }); + test('should log error and set initialized to false if creating/updating common component template throws error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); - expect(createOrUpdateComponentTemplate).toHaveBeenCalledWith( - expect.objectContaining({ - logger, - esClient, - template: expect.objectContaining({ - name: '.risk-score-mappings', - _meta: { - managed: true, - }, - }), - totalFieldsLimit: 1000, - }) + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template .alerts-framework-mappings - fail` ); - expect((createOrUpdateComponentTemplate as jest.Mock).mock.lastCall[0].template.template) - .toMatchInlineSnapshot(` - Object { - "mappings": Object { - "dynamic": "strict", - "properties": Object { - "@timestamp": Object { - "ignore_malformed": false, - "type": "date", - }, - "host": Object { - "properties": Object { - "name": Object { - "type": "keyword", - }, - "risk": Object { - "properties": Object { - "calculated_level": Object { - "type": "keyword", - }, - "calculated_score": Object { - "type": "float", - }, - "calculated_score_norm": Object { - "type": "float", - }, - "category_1_count": Object { - "type": "long", - }, - "category_1_score": Object { - "type": "float", - }, - "id_field": Object { - "type": "keyword", - }, - "id_value": Object { - "type": "keyword", - }, - "inputs": Object { - "properties": Object { - "category": Object { - "type": "keyword", - }, - "description": Object { - "type": "keyword", - }, - "id": Object { - "type": "keyword", - }, - "index": Object { - "type": "keyword", - }, - "risk_score": Object { - "type": "float", - }, - "timestamp": Object { - "type": "date", - }, - }, - "type": "object", - }, - "notes": Object { - "type": "keyword", - }, - }, - "type": "object", - }, - }, - }, - "user": Object { - "properties": Object { - "name": Object { - "type": "keyword", + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + }); + + test('should update index template field limit and retry initialization if creating/updating common component template fails with field limit error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { + error: { + root_cause: [ + { + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', }, - "risk": Object { - "properties": Object { - "calculated_level": Object { - "type": "keyword", - }, - "calculated_score": Object { - "type": "float", - }, - "calculated_score_norm": Object { - "type": "float", - }, - "category_1_count": Object { - "type": "long", - }, - "category_1_score": Object { - "type": "float", - }, - "id_field": Object { - "type": "keyword", - }, - "id_value": Object { - "type": "keyword", - }, - "inputs": Object { - "properties": Object { - "category": Object { - "type": "keyword", - }, - "description": Object { - "type": "keyword", - }, - "id": Object { - "type": "keyword", - }, - "index": Object { - "type": "keyword", - }, - "risk_score": Object { - "type": "float", - }, - "timestamp": Object { - "type": "date", - }, - }, - "type": "object", - }, - "notes": Object { - "type": "keyword", - }, + ], + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', + caused_by: { + type: 'illegal_argument_exception', + reason: 'Limit of total fields [1900] has been exceeded', }, - "type": "object", }, }, }, }, - }, - "settings": Object {}, - } - `); - - expect(createOrUpdateIndexTemplate).toHaveBeenCalledWith({ - logger, - esClient, - template: { - name: '.risk-score.risk-score-default-index-template', - body: { - data_stream: { hidden: true }, - index_patterns: ['risk-score.risk-score-default'], - composed_of: ['.risk-score-mappings'], - template: { - lifecycle: {}, - settings: { - 'index.mapping.total_fields.limit': totalFieldsLimit, - }, - mappings: { - dynamic: false, - _meta: { - kibana: { - version: '8.9.0', - }, - managed: true, - namespace: 'default', - }, + }) + ) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['.alerts-framework-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, }, + 'index.mapping.total_fields.limit': 1800, }, - _meta: { - kibana: { - version: '8.9.0', - }, - managed: true, - namespace: 'default', + mappings: { + dynamic: false, }, }, }, + }; + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], }); - - expect(createDataStream).toHaveBeenCalledWith({ + const assistantService = new AIAssistantService({ logger, - esClient, - totalFieldsLimit, - indexPatterns: { - template: `.risk-score.risk-score-default-index-template`, - alias: `risk-score.risk-score-default`, - }, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - expect(createIndex).toHaveBeenCalledWith({ - logger, - esClient, - options: { - index: `risk-score.risk-score-latest-default`, - mappings: { - dynamic: 'strict', - properties: { - '@timestamp': { - ignore_malformed: false, - type: 'date', - }, - host: { - properties: { - name: { - type: 'keyword', - }, - risk: { - properties: { - calculated_level: { - type: 'keyword', - }, - calculated_score: { - type: 'float', - }, - calculated_score_norm: { - type: 'float', - }, - category_1_count: { - type: 'long', - }, - category_1_score: { - type: 'float', - }, - id_field: { - type: 'keyword', - }, - id_value: { - type: 'keyword', - }, - inputs: { - properties: { - category: { - type: 'keyword', - }, - description: { - type: 'keyword', - }, - id: { - type: 'keyword', - }, - index: { - type: 'keyword', - }, - risk_score: { - type: 'float', - }, - timestamp: { - type: 'date', - }, - }, - type: 'object', - }, - notes: { - type: 'keyword', - }, - }, - type: 'object', - }, - }, - }, - user: { - properties: { - name: { - type: 'keyword', - }, - risk: { - properties: { - calculated_level: { - type: 'keyword', - }, - calculated_score: { - type: 'float', - }, - calculated_score_norm: { - type: 'float', - }, - category_1_count: { - type: 'long', - }, - category_1_score: { - type: 'float', - }, - id_field: { - type: 'keyword', - }, - id_value: { - type: 'keyword', - }, - inputs: { - properties: { - category: { - type: 'keyword', - }, - description: { - type: 'keyword', - }, - id: { - type: 'keyword', - }, - index: { - type: 'keyword', - }, - risk_score: { - type: 'float', - }, - timestamp: { - type: 'date', - }, - }, - type: 'object', - }, - notes: { - type: 'keyword', - }, - }, - type: 'object', - }, - }, - }, + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, }, }, }, }); - expect(transforms.createTransform).toHaveBeenCalledWith({ + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + // 3x for framework, legacy-alert and ecs mappings, then 1 extra time to update component template + // after updating index template field limit + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + }); + }); + + describe('register()', () => { + let assistantService: AIAssistantService; + beforeEach(async () => { + assistantService = new AIAssistantService({ logger, - esClient, - transform: { - dest: { - index: 'risk-score.risk-score-latest-default', - }, - frequency: '1h', - latest: { - sort: '@timestamp', - unique_key: ['host.name', 'user.name'], - }, - source: { - index: ['risk-score.risk-score-default'], - }, - sync: { - time: { - delay: '2s', - field: '@timestamp', - }, - }, - transform_id: 'risk_score_latest_transform_default', - }, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); }); - }); - describe('initializeResources error', () => { - it('should handle errors during initialization', async () => { - const error = new Error('There error'); - (createOrUpdateIndexTemplate as jest.Mock).mockRejectedValueOnce(error); + test('should correctly install resources for context when common initialization is complete', async () => { + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); - try { - await riskEngineDataClient.initializeResources({ namespace: 'default' }); - } catch (e) { - expect(logger.error).toHaveBeenCalledWith( - `Error initializing risk engine resources: ${error.message}` - ); + if (!useDataStreamForAlerts) { + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + } else { + expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); } - }); - }); - describe('getStatus', () => { - it('should return initial status', async () => { - const status = await riskEngineDataClient.getStatus({ - namespace: 'default', - }); - expect(status).toEqual({ - isMaxAmountOfRiskEnginesReached: false, - riskEngineStatus: 'NOT_INSTALLED', - legacyRiskEngineStatus: 'NOT_INSTALLED', - }); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; + expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + getIndexTemplatePutBody({ useDataStream: useDataStreamForAlerts }) + ); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalledWith({ + expand_wildcards: 'all', + name: '.alerts-test.alerts-default', + }); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, + }); + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', + }); + } }); - describe('saved object exists and transforms not', () => { - beforeEach(() => { - mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); - }); + test('should correctly install resources for custom namespace on demand when isSpaceAware is true', async () => { + assistantService.register({ ...TestRegistrationContext, isSpaceAware: true }); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); - it('should return status with enabled true', async () => { - mockSavedObjectClient.find.mockResolvedValue( - getSavedObjectConfiguration({ - enabled: true, - }) - ); + if (!useDataStreamForAlerts) { + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + } else { + expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); + } - const status = await riskEngineDataClient.getStatus({ - namespace: 'default', + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( + 1, + getIndexTemplatePutBody({ useDataStream: useDataStreamForAlerts }) + ); + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenNthCalledWith(1, { + expand_wildcards: 'all', + name: '.alerts-test.alerts-default', + }); + } else { + expect(clusterClient.indices.create).toHaveBeenNthCalledWith(1, { + index: '.internal.alerts-test.alerts-default-000001', + body: { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + }, + }, + }, }); - expect(status).toEqual({ - isMaxAmountOfRiskEnginesReached: true, - riskEngineStatus: 'ENABLED', - legacyRiskEngineStatus: 'NOT_INSTALLED', + expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(1, { + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', }); - }); + } + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); - it('should return status with enabled false', async () => { - mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); + clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ + data_streams: [], + })); + + await retryUntil( + 'context in namespace initialized', + async () => + (await getContextInitialized( + assistantService, + TestRegistrationContext.context, + 'another-namespace' + )) === true + ); - const status = await riskEngineDataClient.getStatus({ - namespace: 'default', + expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( + 2, + getIndexTemplatePutBody({ + namespace: 'another-namespace', + useDataStream: useDataStreamForAlerts, + }) + ); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 4 + ); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 4 + ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 4 + ); + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).toHaveBeenNthCalledWith(1, { + name: '.alerts-test.alerts-another-namespace', }); - expect(status).toEqual({ - isMaxAmountOfRiskEnginesReached: false, - riskEngineStatus: 'DISABLED', - legacyRiskEngineStatus: 'NOT_INSTALLED', + expect(clusterClient.indices.getDataStream).toHaveBeenNthCalledWith(2, { + expand_wildcards: 'all', + name: '.alerts-test.alerts-another-namespace', }); - }); + } else { + expect(clusterClient.indices.create).toHaveBeenNthCalledWith(2, { + index: '.internal.alerts-test.alerts-another-namespace-000001', + body: { + aliases: { + '.alerts-test.alerts-another-namespace': { + is_write_index: true, + }, + }, + }, + }); + expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(2, { + index: '.internal.alerts-test.alerts-another-namespace-*', + name: '.alerts-test.alerts-*', + }); + } }); - describe('legacy transforms', () => { - it('should fetch transforms', async () => { - await riskEngineDataClient.getStatus({ - namespace: 'default', - }); + test('should not install component template for context if fieldMap is empty', async () => { + assistantService.register({ + context: 'empty', + mappings: { fieldMap: {} }, + }); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService, 'empty')) === true + ); - expect(esClient.transform.getTransform).toHaveBeenCalledTimes(4); - expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(1, { - transform_id: 'ml_hostriskscore_pivot_transform_default', - }); - expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(2, { - transform_id: 'ml_hostriskscore_latest_transform_default', + if (!useDataStreamForAlerts) { + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + } else { + expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); + } + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + + const template = { + name: `.alerts-empty.alerts-default-index-template`, + body: { + index_patterns: [ + useDataStreamForAlerts + ? `.alerts-empty.alerts-default` + : `.internal.alerts-empty.alerts-default-*`, + ], + composed_of: ['.alerts-framework-mappings'], + ...(useDataStreamForAlerts ? { data_stream: { hidden: true } } : {}), + priority: 7, + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + ...(useDataStreamForAlerts + ? {} + : { + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty.alerts-default`, + }, + }), + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': 2500, + }, + mappings: { + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace: 'default', + }, + dynamic: false, + }, + }, + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace: 'default', + }, + }, + }; + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith(template); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalledWith({}); + expect(clusterClient.indices.getDataStream).toHaveBeenCalledWith({ + expand_wildcards: 'all', + name: '.alerts-empty.alerts-default', }); - expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(3, { - transform_id: 'ml_userriskscore_pivot_transform_default', + } else { + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-empty.alerts-default-000001', + body: { + aliases: { + '.alerts-empty.alerts-default': { + is_write_index: true, + }, + }, + }, }); - expect(esClient.transform.getTransform).toHaveBeenNthCalledWith(4, { - transform_id: 'ml_userriskscore_latest_transform_default', + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.internal.alerts-empty.alerts-default-*', + name: '.alerts-empty.alerts-*', }); - }); + } + + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 1 : 2 + ); + }); - it('should return that legacy transform enabled if at least on transform exist', async () => { - esClient.transform.getTransform.mockResolvedValueOnce(transformsMock); + test('should skip initialization if context already exists', async () => { + assistantService.register(TestRegistrationContext); + assistantService.register(TestRegistrationContext); - const status = await riskEngineDataClient.getStatus({ - namespace: 'default', + expect(logger.debug).toHaveBeenCalledWith( + `Resources for context "test" have already been registered.` + ); + }); + + test('should throw error if context already exists and has been registered with a different field map', async () => { + assistantService.register(TestRegistrationContext); + expect(() => { + assistantService.register({ + ...TestRegistrationContext, + mappings: { fieldMap: { anotherField: { type: 'keyword', required: false } } }, }); + }).toThrowErrorMatchingInlineSnapshot( + `"test has already been registered with different options"` + ); + }); - expect(status).toEqual({ - isMaxAmountOfRiskEnginesReached: false, - riskEngineStatus: 'NOT_INSTALLED', - legacyRiskEngineStatus: 'ENABLED', + test('should throw error if context already exists and has been registered with a different options', async () => { + assistantService.register(TestRegistrationContext); + expect(() => { + assistantService.register({ + ...TestRegistrationContext, + useEcs: true, }); + }).toThrowErrorMatchingInlineSnapshot( + `"test has already been registered with different options"` + ); + }); - esClient.transform.getTransformStats.mockReset(); + test('should allow same context with different "shouldWrite" option', async () => { + assistantService.register(TestRegistrationContext); + assistantService.register({ + ...TestRegistrationContext, + shouldWrite: false, }); + + expect(logger.debug).toHaveBeenCalledWith( + `Resources for context "test" have already been registered.` + ); }); - }); - describe('#getConfiguration', () => { - it('retrieves configuration from the saved object', async () => { - mockSavedObjectClient.find.mockResolvedValueOnce(getSavedObjectConfiguration()); + test('should not update index template if simulating template throws error', async () => { + clusterClient.indices.simulateTemplate.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); - const configuration = await riskEngineDataClient.getConfiguration(); + expect(logger.error).toHaveBeenCalledWith( + `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - fail`, + expect.any(Error) + ); - expect(mockSavedObjectClient.find).toHaveBeenCalledTimes(1); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + // putIndexTemplate is skipped but other operations are called as expected + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } + }); - expect(configuration).toEqual({ - enabled: false, + test('should log error and set initialized to false if simulating template returns empty mappings', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ + result: false, + error: + 'Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping', }); + + expect(logger.error).toHaveBeenCalledWith( + new Error( + `No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping` + ) + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); }); - }); - describe('enableRiskEngine', () => { - let mockTaskManagerStart: ReturnType; + test('should log error and set initialized to false if updating index template throws error', async () => { + clusterClient.indices.putIndexTemplate.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - beforeEach(() => { - mockSavedObjectClient.find.mockResolvedValue(getSavedObjectConfiguration()); - mockTaskManagerStart = taskManagerMock.createStart(); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ error: 'Failure during installation. fail', result: false }); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test.alerts-default-index-template - fail`, + expect.any(Error) + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); }); - it('returns an error if saved object does not exist', async () => { - mockSavedObjectClient.find.mockResolvedValue({ - page: 1, - per_page: 20, - total: 0, - saved_objects: [], - }); + test('should log error and set initialized to false if checking for concrete write index throws error', async () => { + clusterClient.indices.getAlias.mockRejectedValueOnce(new Error('fail')); + clusterClient.indices.getDataStream.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ error: 'Failure during installation. fail', result: false }); + + expect(logger.error).toHaveBeenCalledWith( + useDataStreamForAlerts + ? `Error fetching data stream for .alerts-test.alerts-default - fail` + : `Error fetching concrete indices for .internal.alerts-test.alerts-default-* pattern - fail` + ); - await expect( - riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }) - ).rejects.toThrow('Risk engine configuration not found'); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); }); - it('should update saved object attribute', async () => { - await riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }); + test('should not throw error if checking for concrete write index throws 404', async () => { + const error = new Error(`index doesn't exist`) as HTTPError; + error.statusCode = 404; + clusterClient.indices.getAlias.mockRejectedValueOnce(error); + clusterClient.indices.getDataStream.mockRejectedValueOnce(error); - expect(mockSavedObjectClient.update).toHaveBeenCalledWith( - 'risk-engine-configuration', - 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', - { - enabled: true, - }, - { - refresh: 'wait_for', - } + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + } }); - describe('if task manager throws an error', () => { - beforeEach(() => { - mockTaskManagerStart.ensureScheduled.mockRejectedValueOnce( - new Error('Task Manager error') - ); - }); + test('should log error and set initialized to false if updating index settings for existing indices throws error', async () => { + clusterClient.indices.putSettings.mockRejectedValueOnce(new Error('fail')); - it('disables the risk engine and re-throws the error', async () => { - await expect( - riskEngineDataClient.enableRiskEngine({ taskManager: mockTaskManagerStart }) - ).rejects.toThrow('Task Manager error'); + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect(mockSavedObjectClient.update).toHaveBeenCalledWith( - 'risk-engine-configuration', - 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', - { - enabled: false, - }, - { - refresh: 'wait_for', - } + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ error: 'Failure during installation. fail', result: false }); + + expect(logger.error).toHaveBeenCalledWith( + useDataStreamForAlerts + ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: fail` + : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } + }); + + test('should skip updating index mapping for existing indices if simulate index template throws error', async () => { + clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(logger.error).toHaveBeenCalledWith( + useDataStreamForAlerts + ? `Ignored PUT mappings for .alerts-test.alerts-default; error generating simulated mappings: fail` + : `Ignored PUT mappings for alias_1; error generating simulated mappings: fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + + // this is called to update backing indices, so not used with data streams + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + } + }); + + test('should log error and set initialized to false if updating index mappings for existing indices throws error', async () => { + clusterClient.indices.putMapping.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ error: 'Failure during installation. fail', result: false }); + + if (useDataStreamForAlerts) { + expect(logger.error).toHaveBeenCalledWith( + `Failed to PUT mapping for .alerts-test.alerts-default: fail` ); - }); + } else { + expect(logger.error).toHaveBeenCalledWith(`Failed to PUT mapping for alias_1: fail`); + } + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } }); - }); - describe('disableRiskEngine', () => { - let mockTaskManagerStart: ReturnType; + test('does not updating settings or mappings if no existing concrete indices', async () => { + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({})); + clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ + data_streams: [], + })); - beforeEach(() => { - mockTaskManagerStart = taskManagerMock.createStart(); + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } }); - it('should return error if saved object not exist', async () => { - mockSavedObjectClient.find.mockResolvedValueOnce({ - page: 1, - per_page: 20, - total: 0, - saved_objects: [], + test('should log error and set initialized to false if concrete indices exist but none are write index', async () => { + // not applicable for data streams + if (useDataStreamForAlerts) return; + + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-0001': { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: false, + is_hidden: true, + }, + alias_2: { + is_write_index: false, + is_hidden: true, + }, + }, + }, + })); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ + error: + 'Failure during installation. Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default', + result: false, }); - expect.assertions(1); - try { - await riskEngineDataClient.disableRiskEngine({ taskManager: mockTaskManagerStart }); - } catch (e) { - expect(e.message).toEqual('Risk engine configuration not found'); + expect(logger.error).toHaveBeenCalledWith( + new Error( + `Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default` + ) + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('does not create new index if concrete write index exists', async () => { + // not applicable for data streams + if (useDataStreamForAlerts) return; + + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-0001': { + aliases: { + '.alerts-test.alerts-default': { + is_write_index: true, + is_hidden: true, + }, + alias_2: { + is_write_index: false, + is_hidden: true, + }, + }, + }, + })); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); } }); - it('should update saved object attrubute', async () => { - mockSavedObjectClient.find.mockResolvedValueOnce(getSavedObjectConfiguration()); + test('should log error and set initialized to false if create concrete index throws error', async () => { + // not applicable for data streams + if (useDataStreamForAlerts) return; + + clusterClient.indices.create.mockRejectedValueOnce(new Error('fail')); + clusterClient.indices.createDataStream.mockRejectedValueOnce(new Error('fail')); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ error: 'Failure during installation. fail', result: false }); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } + }); - await riskEngineDataClient.disableRiskEngine({ taskManager: mockTaskManagerStart }); + test('should not throw error if create concrete index throws resource_already_exists_exception error and write index already exists', async () => { + // not applicable for data streams + if (useDataStreamForAlerts) return; - expect(mockSavedObjectClient.update).toHaveBeenCalledWith( - 'risk-engine-configuration', - 'de8ca330-2d26-11ee-bc86-f95bf6192ee6', - { - enabled: false, + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, }, - { - refresh: 'wait_for', - } + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, + }, + })); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true ); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.get).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if create concrete index throws resource_already_exists_exception error and write index does not already exists', async () => { + // not applicable for data streams + if (useDataStreamForAlerts) return; + + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: false } }, + }, + })); + + assistantService.register(TestRegistrationContext); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await assistantService.getContextInitializationPromise( + TestRegistrationContext.context, + DEFAULT_NAMESPACE_STRING + ) + ).toEqual({ + error: + 'Failure during installation. Attempted to create index: .internal.alerts-test.alerts-default-000001 as the write index for alias: .alerts-test.alerts-default, but the index already exists and is not the write index for the alias', + result: false, + }); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.get).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); }); }); - describe('init', () => { - let mockTaskManagerStart: ReturnType; - const initializeResourcesMock = jest.spyOn( - RiskEngineDataClient.prototype, - 'initializeResources' - ); - const enableRiskEngineMock = jest.spyOn(RiskEngineDataClient.prototype, 'enableRiskEngine'); + describe('createAIAssistantDatastreamClient()', () => { + let assistantService: AIAssistantService; + beforeEach(async () => { + (AIAssistantConversationsDataClient as jest.Mock).mockImplementation( + () => conversationsDataClient + ); + }); - const disableLegacyRiskEngineMock = jest.spyOn( - RiskEngineDataClient.prototype, - 'disableLegacyRiskEngine' - ); - beforeEach(() => { - mockTaskManagerStart = taskManagerMock.createStart(); - disableLegacyRiskEngineMock.mockImplementation(() => Promise.resolve(true)); + test('should create new AlertsClient', async () => { + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }); - initializeResourcesMock.mockImplementation(() => { - return Promise.resolve(); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + dataStreamAdapter, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + kibanaVersion: '8.8.0', }); + }); + + test('should retry initializing common resources if common resource initialization failed', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + assistantService.register(TestRegistrationContext); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + const result = await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 2 + ); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } - enableRiskEngineMock.mockImplementation(() => { - return Promise.resolve(getSavedObjectConfiguration().saved_objects[0]); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + dataStreamAdapter, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + kibanaVersion: '8.8.0', }); - jest - .spyOn(savedObjectConfig, 'initSavedObjects') - .mockResolvedValue({} as unknown as SavedObject); + expect(result).not.toBe(null); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.info).toHaveBeenCalledWith( + `Resource installation for "test" succeeded after retry` + ); }); - afterEach(() => { - initializeResourcesMock.mockReset(); - enableRiskEngineMock.mockReset(); - disableLegacyRiskEngineMock.mockReset(); + test('should not retry initializing common resources if common resource initialization is in progress', async () => { + // this is the initial call that fails + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); + + // this is the retry call that we'll artificially inflate the duration of + clusterClient.cluster.putComponentTemplate.mockImplementationOnce(async () => { + await new Promise((r) => setTimeout(r, 1000)); + return { acknowledged: true }; + }); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + assistantService.register(TestRegistrationContext); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + // call createAIAssistantConversationsDataClient at the same time which will trigger the retries + const result = await Promise.all([ + assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }), + assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }), + ]); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 2 + ); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + } else { + expect(clusterClient.indices.create).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + } + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + dataStreamAdapter, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + kibanaVersion: '8.8.0', + }); + + expect(result[0]).not.toBe(null); + expect(result[1]).not.toBe(null); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.info).toHaveBeenCalledWith( + `Resource installation for "test" succeeded after retry` + ); + expect(logger.info).toHaveBeenCalledWith( + `Skipped retrying common resource initialization because it is already being retried.` + ); }); - it('success', async () => { - const initResult = await riskEngineDataClient.init({ + test('should retry initializing context specific resources if context specific resource initialization failed', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + assistantService.register(TestRegistrationContext); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + const result = await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, namespace: 'default', - taskManager: mockTaskManagerStart, + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, }); - expect(initResult).toEqual({ - errors: [], - legacyRiskEngineDisabled: true, - riskEngineConfigurationCreated: true, - riskEngineEnabled: true, - riskEngineResourcesInstalled: true, + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + dataStreamAdapter, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + kibanaVersion: '8.8.0', }); + + expect(result).not.toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.info).toHaveBeenCalledWith( + `Resource installation for "test" succeeded after retry` + ); }); - it('should catch error for disableLegacyRiskEngine, but continue', async () => { - disableLegacyRiskEngineMock.mockImplementation(() => { - throw new Error('Error disableLegacyRiskEngineMock'); + test('should not retry initializing context specific resources if context specific resource initialization is in progress', async () => { + // this is the initial call that fails + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + // this is the retry call that we'll artificially inflate the duration of + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => { + await new Promise((r) => setTimeout(r, 1000)); + return SimulateTemplateResponse; + }); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - const initResult = await riskEngineDataClient.init({ + assistantService.register(TestRegistrationContext); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + const createAlertsClientWithDelay = async (delayMs: number | null) => { + if (delayMs) { + await new Promise((r) => setTimeout(r, delayMs)); + } + + return await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }); + }; + + const result = await Promise.all([ + createAlertsClientWithDelay(null), + createAlertsClientWithDelay(1), + ]); + + expect(AIAssistantConversationsDataClient).toHaveBeenCalledTimes(2); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + dataStreamAdapter, + ruleType: ruleTypeWithAlertDefinition, namespace: 'default', - taskManager: mockTaskManagerStart, + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + kibanaVersion: '8.8.0', }); - expect(initResult).toEqual({ - errors: ['Error disableLegacyRiskEngineMock'], - legacyRiskEngineDisabled: false, - riskEngineConfigurationCreated: true, - riskEngineEnabled: true, - riskEngineResourcesInstalled: true, + expect(result[0]).not.toBe(null); + expect(result[1]).not.toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + + // Should only log the retry once because the second call should + // leverage the outcome of the first retry + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + ).length + ).toEqual(1); + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => + calls[0] === `Resource installation for "test" succeeded after retry` + ).length + ).toEqual(1); + }); + + test('should throttle retries of initializing context specific resources', async () => { + // this is the initial call that fails + clusterClient.indices.simulateTemplate.mockImplementation(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + assistantService.register(TestRegistrationContext); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + const createAlertsClientWithDelay = async (delayMs: number | null) => { + if (delayMs) { + await new Promise((r) => setTimeout(r, delayMs)); + } + + return await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }); + }; + + await Promise.all([ + createAlertsClientWithDelay(null), + createAlertsClientWithDelay(1), + createAlertsClientWithDelay(2), + ]); + + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + + // Should only log the retry once because the second and third retries should be throttled + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + ).length + ).toEqual(1); }); - it('should catch error for resource init', async () => { - disableLegacyRiskEngineMock.mockImplementationOnce(() => { - throw new Error('Error disableLegacyRiskEngineMock'); + test('should return null if retrying common resources initialization fails again', async () => { + let failCount = 0; + clusterClient.cluster.putComponentTemplate.mockImplementation(() => { + throw new Error(`fail ${++failCount}`); }); - const initResult = await riskEngineDataClient.init({ + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, + }); + assistantService.register(TestRegistrationContext); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + const result = await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, namespace: 'default', - taskManager: mockTaskManagerStart, + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, }); - expect(initResult).toEqual({ - errors: ['Error disableLegacyRiskEngineMock'], - legacyRiskEngineDisabled: false, - riskEngineConfigurationCreated: true, - riskEngineEnabled: true, - riskEngineResourcesInstalled: true, + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 2 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + expect(result).toBe(null); + expect(AlertsClient).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringMatching( + /There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation\. fail \d+; Error after retry: Failure during installation\. fail \d+/ + ) + ); + }); + + test('should return null if retrying common resources initialization fails again with same error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('fail')); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + assistantService.register(TestRegistrationContext); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing component template failed so no calls to install context-specific resources + // should be made + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 1 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + const result = await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, + namespace: 'default', + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, + }); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 0 : 2 + ); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + expect(result).toBe(null); + expect(AlertsClient).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.warn).toHaveBeenCalledWith( + `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Retry failed with error: Failure during installation. fail` + ); }); - it('should catch error for initializeResources and stop', async () => { - initializeResourcesMock.mockImplementationOnce(() => { - throw new Error('Error initializeResourcesMock'); + test('should return null if retrying context specific initialization fails again', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + clusterClient.indices.putIndexTemplate.mockRejectedValueOnce( + new Error('fail index template') + ); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + assistantService.register(TestRegistrationContext); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); - const initResult = await riskEngineDataClient.init({ + const result = await assistantService.createAlertsClient({ + logger, + ruleType: ruleTypeWithAlertDefinition, namespace: 'default', - taskManager: mockTaskManagerStart, + rule: { + consumer: 'bar', + executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + id: '1', + name: 'rule-name', + parameters: { + bar: true, + }, + revision: 0, + spaceId: 'default', + tags: ['rule-', '-tags'], + }, }); - expect(initResult).toEqual({ - errors: ['Error initializeResourcesMock'], - legacyRiskEngineDisabled: true, - riskEngineConfigurationCreated: false, - riskEngineEnabled: false, - riskEngineResourcesInstalled: false, + expect(AlertsClient).not.toHaveBeenCalled(); + expect(result).toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.warn).toHaveBeenCalledWith( + `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping; Error after retry: Failure during installation. fail index template` + ); + }); + }); + + describe('retries', () => { + test('should retry adding ILM policy for transient ES errors', async () => { + if (useDataStreamForAlerts) return; + + clusterClient.ilm.putLifecycle + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(3); }); - it('should catch error for initSavedObjects and stop', async () => { - jest.spyOn(savedObjectConfig, 'initSavedObjects').mockImplementationOnce(() => { - throw new Error('Error initSavedObjects'); + test('should retry adding component template for transient ES errors', async () => { + clusterClient.cluster.putComponentTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - const initResult = await riskEngineDataClient.init({ - namespace: 'default', - taskManager: mockTaskManagerStart, + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); + }); + + test('should retry updating index template for transient ES errors', async () => { + clusterClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - expect(initResult).toEqual({ - errors: ['Error initSavedObjects'], - legacyRiskEngineDisabled: true, - riskEngineConfigurationCreated: false, - riskEngineEnabled: false, - riskEngineResourcesInstalled: true, + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + expect(assistantService.isInitialized()).toEqual(true); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index settings for existing indices for transient ES errors', async () => { + clusterClient.indices.putSettings + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); + + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(3); + } else { + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); + } }); - it('should catch error for enableRiskEngineMock and stop', async () => { - enableRiskEngineMock.mockImplementationOnce(() => { - throw new Error('Error enableRiskEngineMock'); + test('should retry updating index mappings for existing indices for transient ES errors', async () => { + clusterClient.indices.putMapping + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - const initResult = await riskEngineDataClient.init({ - namespace: 'default', - taskManager: mockTaskManagerStart, + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( + useDataStreamForAlerts ? 3 : 4 + ); + }); + + test('should retry creating concrete index for transient ES errors', async () => { + clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ + data_streams: [], + })); + clusterClient.indices.createDataStream + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + clusterClient.indices.create + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ index: 'index', shards_acknowledged: true, acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + dataStreamAdapter, }); - expect(initResult).toEqual({ - errors: ['Error enableRiskEngineMock'], - legacyRiskEngineDisabled: true, - riskEngineConfigurationCreated: true, - riskEngineEnabled: false, - riskEngineResourcesInstalled: true, + await retryUntil( + 'alert service initialized', + async () => assistantService.isInitialized() === true + ); + + assistantService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(assistantService)) === true + ); + + if (useDataStreamForAlerts) { + expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); + } else { + expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); + } + }); + }); + + describe('timeout', () => { + test('should short circuit initialization if timeout exceeded', async () => { + clusterClient.cluster.putComponentTemplate.mockImplementationOnce(async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return { acknowledged: true }; }); + new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + timeoutMs: 10, + dataStreamAdapter, + }); + + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + + expect(logger.error).toHaveBeenCalledWith(new Error(`Timeout: it took more than 10ms`)); + }); + + test('should short circuit initialization if pluginStop$ signal received but not throw error', async () => { + pluginStop$.next(); + new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + timeoutMs: 10, + dataStreamAdapter, + }); + + await retryUntil('debug logger called', async () => logger.debug.mock.calls.length > 0); + + expect(logger.debug).toHaveBeenCalledWith( + `Server is stopping; must stop all async operations` + ); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index 5b177a0a7677c..dd51f24c23802 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -37,7 +37,7 @@ export const appendConversationMessages = async ( values: [existingConversation.id ?? ''], }, }, - refresh: false, + refresh: true, script: { lang: 'painless', params: { @@ -77,7 +77,12 @@ export const appendConversationMessages = async ( ); throw err; } - return getConversation(esClient, conversationIndex, existingConversation.id ?? ''); + const updatedConversation = await getConversation( + esClient, + conversationIndex, + existingConversation.id ?? '' + ); + return updatedConversation; }; export const transformToUpdateScheme = (updatedAt: string, messages: Message[]) => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts new file mode 100644 index 0000000000000..271d5511e6b75 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts @@ -0,0 +1,345 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { RiskEngineDataWriter } from './risk_engine_data_writer'; +import { riskScoreServiceMock } from './risk_score_service.mock'; + +describe('RiskEngineDataWriter', () => { + describe('#bulk', () => { + let writer: RiskEngineDataWriter; + let esClientMock: ElasticsearchClient; + let loggerMock: Logger; + + beforeEach(() => { + esClientMock = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; + loggerMock = loggingSystemMock.createLogger(); + writer = new RiskEngineDataWriter({ + esClient: esClientMock, + logger: loggerMock, + index: 'risk-score.risk-score-default', + namespace: 'default', + }); + }); + + it('converts a list of host risk scores to an appropriate list of operations', async () => { + await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore(), riskScoreServiceMock.createRiskScore()], + }); + + const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; + + expect(operations).toMatchInlineSnapshot(` + Array [ + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "host": Object { + "name": "hostname", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "host.name", + "id_value": "hostname", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "host": Object { + "name": "hostname", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "host.name", + "id_value": "hostname", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + ] + `); + }); + + it('converts a list of user risk scores to an appropriate list of operations', async () => { + await writer.bulk({ + user: [ + riskScoreServiceMock.createRiskScore({ + id_field: 'user.name', + id_value: 'username_1', + }), + riskScoreServiceMock.createRiskScore({ + id_field: 'user.name', + id_value: 'username_2', + }), + ], + }); + + const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; + + expect(operations).toMatchInlineSnapshot(` + Array [ + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "user": Object { + "name": "username_1", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "user.name", + "id_value": "username_1", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "user": Object { + "name": "username_2", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "user.name", + "id_value": "username_2", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + ] + `); + }); + + it('converts a list of mixed risk scores to an appropriate list of operations', async () => { + await writer.bulk({ + host: [ + riskScoreServiceMock.createRiskScore({ + id_field: 'host.name', + id_value: 'hostname_1', + }), + ], + user: [ + riskScoreServiceMock.createRiskScore({ + id_field: 'user.name', + id_value: 'username_1', + }), + riskScoreServiceMock.createRiskScore({ + id_field: 'user.name', + id_value: 'username_2', + }), + ], + }); + + const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; + + expect(operations).toMatchInlineSnapshot(` + Array [ + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "host": Object { + "name": "hostname_1", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "host.name", + "id_value": "hostname_1", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "user": Object { + "name": "username_1", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "user.name", + "id_value": "username_1", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + Object { + "create": Object { + "_index": "risk-score.risk-score-default", + }, + }, + Object { + "@timestamp": "2023-02-15T00:15:19.231Z", + "user": Object { + "name": "username_2", + "risk": Object { + "calculated_level": "High", + "calculated_score": 149, + "calculated_score_norm": 85.332, + "category_1_count": 12, + "category_1_score": 85, + "category_2_count": 0, + "category_2_score": 0, + "criticality_level": "very_important", + "criticality_modifier": 2, + "id_field": "user.name", + "id_value": "username_2", + "inputs": Array [], + "notes": Array [], + }, + }, + }, + ] + `); + }); + + it('returns an error if something went wrong', async () => { + (esClientMock.bulk as jest.Mock).mockRejectedValue(new Error('something went wrong')); + + const { errors } = await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore()], + }); + + expect(errors).toEqual(['something went wrong']); + }); + + it('returns the time it took to write the risk scores', async () => { + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + took: 123, + items: [], + }); + + const { took } = await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore()], + }); + + expect(took).toEqual(123); + }); + + it('returns the number of docs written', async () => { + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + items: [{ create: { status: 201 } }, { create: { status: 200 } }], + }); + + const { docs_written: docsWritten } = await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore()], + }); + + expect(docsWritten).toEqual(2); + }); + + describe('when some documents failed to be written', () => { + beforeEach(() => { + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + errors: true, + items: [ + { create: { status: 201 } }, + { create: { status: 500, error: { reason: 'something went wrong' } } }, + ], + }); + }); + + it('returns the number of docs written', async () => { + const { docs_written: docsWritten } = await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore()], + }); + + expect(docsWritten).toEqual(1); + }); + + it('returns the errors', async () => { + const { errors } = await writer.bulk({ + host: [riskScoreServiceMock.createRiskScore()], + }); + + expect(errors).toEqual(['something went wrong']); + }); + }); + + describe('when there are no risk scores to write', () => { + it('returns an appropriate response', async () => { + const response = await writer.bulk({}); + expect(response).toEqual({ errors: [], docs_written: 0, took: 0 }); + }); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index c183ef623e453..f3ae1e8ecc646 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { v4 as uuidV4 } from 'uuid'; import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import { @@ -99,7 +100,7 @@ export class ConversationDataWriter implements ConversationDataWriter { const changedAt = new Date().toISOString(); const conversationBody = params.conversationsToCreate?.flatMap((conversation) => [ - { create: { _index: this.options.index, op_type: 'create' } }, + { create: { _index: this.options.index, _id: uuidV4() } }, transformToCreateScheme(changedAt, this.options.spaceId, this.options.user, conversation), ]) ?? []; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts new file mode 100644 index 0000000000000..f125d45c291ce --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { createConversation } from './create_conversation'; + +describe('createConversation', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list as expected with the id changed out for the elastic id', async () => { + const options = getCreateListOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.create.mockResponse( + // @ts-expect-error not full response interface + { _id: 'elastic-id-123' } + ); + const list = await createConversation({ ...options, esClient }); + const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; + expect(list).toEqual(expected); + }); + + test('it returns a list as expected with the id changed out for the elastic id and seralizer and deseralizer set', async () => { + const options: CreateListOptions = { + ...getCreateListOptionsMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.create.mockResponse( + // @ts-expect-error not full response interface + { _id: 'elastic-id-123' } + ); + const list = await createConversation({ ...options, esClient }); + const expected: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + id: 'elastic-id-123', + serializer: '(?)', + }; + expect(list).toEqual(expected); + }); + + test('It calls "esClient" with body, id, and conversationIndex', async () => { + const options = getCreateListOptionsMock(); + await createConversation(options); + const body = getIndexESListMock(); + const expected = { + body, + id: LIST_ID, + index: LIST_INDEX, + refresh: 'wait_for', + }; + expect(options.esClient.create).toBeCalledWith(expected); + }); + + test('It returns an auto-generated id if id is sent in undefined', async () => { + const options = getCreateListOptionsMock(); + options.id = undefined; + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.create.mockResponse( + // @ts-expect-error not full response interface + { _id: 'elastic-id-123' } + ); + const list = await createConversation({ ...options, esClient }); + const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; + expect(list).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts new file mode 100644 index 0000000000000..dd2a8639e9278 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getListResponseMock } from '../../../common/schemas/response/list_schema.mock'; +import { LIST_ID, LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; + +import { getList } from './get_list'; +import { deleteList } from './delete_list'; +import { getDeleteListOptionsMock } from './delete_list.mock'; + +jest.mock('../utils', () => ({ + waitUntilDocumentIndexed: jest.fn(), +})); + +jest.mock('./get_list', () => ({ + getList: jest.fn(), +})); + +describe('delete_list', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Delete returns a null if the list is also null', async () => { + (getList as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteListOptionsMock(); + const deletedList = await deleteList(options); + expect(deletedList).toEqual(null); + }); + + test('Delete returns the list if a list is returned from getList', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); + const deletedList = await deleteList(options); + expect(deletedList).toEqual(list); + }); + + test('Delete calls "deleteByQuery" for list items if a list is returned from getList', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); + await deleteList(options); + const deleteByQuery = { + body: { query: { term: { list_id: LIST_ID } } }, + conflicts: 'proceed', + index: LIST_ITEM_INDEX, + refresh: false, + }; + expect(options.esClient.deleteByQuery).toHaveBeenNthCalledWith(1, deleteByQuery); + }); + + test('Delete calls "deleteByQuery" for list if a list is returned from getList', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); + await deleteList(options); + const deleteByQuery = { + body: { + query: { + ids: { + values: [LIST_ID], + }, + }, + }, + conflicts: 'proceed', + index: LIST_INDEX, + refresh: false, + }; + expect(options.esClient.deleteByQuery).toHaveBeenCalledWith(deleteByQuery); + }); + + test('Delete does not call data client if the list returns null', async () => { + (getList as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteListOptionsMock(); + await deleteList(options); + expect(options.esClient.delete).not.toHaveBeenCalled(); + }); + + test('throw error if no list was deleted', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 0 }); + + await expect(deleteList(options)).rejects.toThrow('No list has been deleted'); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts new file mode 100644 index 0000000000000..c8f10acf556ba --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { getConversation } from './get_conversation'; + +describe('getConversation', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list as expected if the list is found', async () => { + const data = getSearchListMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.search.mockResponse(data); + const conversation = await getConversation(esClient, LIST_INDEX, id ); + const expected = getListResponseMock(); + expect(conversation).toEqual(expected); + }); + + test('it returns null if the search is empty', async () => { + const data = getSearchListMock(); + data.hits.hits = []; + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.search.mockResponse(data); + const conversation = await getConversation(esClient, LIST_INDEX, id); + expect(conversation).toEqual(null); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts new file mode 100644 index 0000000000000..5d18ed8bfa0f6 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; +import { LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; + +import { getListClientMock } from './list_client.mock'; + +describe('AIAssistantConversationsDataClient', () => { + describe('Mock client checks (not exhaustive tests against it)', () => { + test('it returns the get list index as expected', () => { + const mock = getListClientMock(); + expect(mock.getListName()).toEqual(LIST_INDEX); + }); + + test('it returns the get list item index as expected', () => { + const mock = getListClientMock(); + expect(mock.getListItemName()).toEqual(LIST_ITEM_INDEX); + }); + + test('it returns a mock list item', async () => { + const mock = getListClientMock(); + const listItem = await mock.getListItem({ id: '123' }); + expect(listItem).toEqual(getListItemResponseMock()); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 1480ec5af5999..a8698053beedd 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -119,15 +119,11 @@ export class AIAssistantConversationsDataClient { }; /** - * Creates a conversation, if given at least the "title" and "apiConfig" - * See {@link https://www.elastic.co/guide/en/security/current/} - * for more information around formats of the deserializer and serializer + * Updates a conversation with the new messages. * @param options - * @param options.id The id of the conversat to create or "undefined" if you want an "id" to be auto-created for you - * @param options.title A custom deserializer for the conversation. Optionally, you an define this as handle bars. See online docs for more information. + * @param options.conversation The existing conversation to which append the new messages. * @param options.messages Set this to true if this is a conversation that is "immutable"/"pre-packaged". - * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. - * @returns The conversation created + * @returns The conversation updated */ public appendConversationMessages = async ( conversation: ConversationResponse, @@ -178,7 +174,7 @@ export class AIAssistantConversationsDataClient { * See {@link https://www.elastic.co/guide/en/security/current/} * for more information around formats of the deserializer and serializer * @param options - * @param options.id The id of the conversat to create or "undefined" if you want an "id" to be auto-created for you + * @param options.id The id of the conversation to create or "undefined" if you want an "id" to be auto-created for you * @param options.title A custom deserializer for the conversation. Optionally, you an define this as handle bars. See online docs for more information. * @param options.messages Set this to true if this is a conversation that is "immutable"/"pre-packaged". * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. @@ -203,12 +199,11 @@ export class AIAssistantConversationsDataClient { * See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html} * for more information around optimistic concurrency control. * @param options - * @param options._version This is the version, useful for optimistic concurrency control. * @param options.id id of the conversation to replace the conversation container data with. - * @param options.name The new name, or "undefined" if this should not be updated. - * @param options.description The new description, or "undefined" if this should not be updated. - * @param options.meta Additional meta data to associate with the conversation items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. - * @param options.version Updates the version of the conversation. + * @param options.title The new tilet, or "undefined" if this should not be updated. + * @param options.messages The new messages, or "undefined" if this should not be updated. + * @param options.excludeFromLastConversationStorage The new value for excludeFromLastConversationStorage, or "undefined" if this should not be updated. + * @param options.replacements The new value for replacements, or "undefined" if this should not be updated. */ public updateConversation = async ( existingConversation: ConversationResponse, @@ -218,6 +213,7 @@ export class AIAssistantConversationsDataClient { const esClient = await this.options.elasticsearchClientPromise; return updateConversation( esClient, + this.options.logger, this.indexTemplateAndPattern.alias, existingConversation, updatedProps, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts new file mode 100644 index 0000000000000..8d799441291b1 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { updateConversation } from './update_conversation'; + +jest.mock('./get_conversation', () => ({ + getList: jest.fn(), +})); + +describe('updateConversation', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list with serializer and deserializer', async () => { + const list: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getupdateConversationOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResolvedValue({ updated: 1 }); + const updatedList = await updateConversation({ ...options, esClient }); + const expected: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + id: list.id, + serializer: '(?)', + }; + expect(updatedList).toEqual(expected); + }); + + test('it returns null when there is not a list to update', async () => { + (getList as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getupdateConversationOptionsMock(); + const updatedList = await updateConversation(options); + expect(updatedList).toEqual(null); + }); + + test('throw error if no list was updated', async () => { + const list: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getupdateConversationOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResolvedValue({ updated: 0 }); + await expect(updateConversation({ ...options, esClient })).rejects.toThrow('No list has been updated'); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 2dd0d81742025..47a2aa5184db6 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -5,12 +5,11 @@ * 2.0. */ -import { ElasticsearchClient } from '@kbn/core/server'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { ConversationResponse, ConversationUpdateProps, } from '../schemas/conversations/common_attributes.gen'; -import { waitUntilDocumentIndexed } from '../lib/wait_until_document_indexed'; import { getConversation } from './get_conversation'; export interface UpdateConversationSchema { @@ -46,36 +45,34 @@ export interface UpdateConversationSchema { export const updateConversation = async ( esClient: ElasticsearchClient, + logger: Logger, conversationIndex: string, existingConversation: ConversationResponse, conversation: ConversationUpdateProps, isPatch?: boolean ): Promise => { const updatedAt = new Date().toISOString(); + const params = transformToUpdateScheme(updatedAt, conversation); - // checkVersionConflict(_version, list._version); - // const calculatedVersion = version == null ? list.version + 1 : version; - - const params: UpdateConversationSchema = transformToUpdateScheme(updatedAt, conversation); - - const response = await esClient.updateByQuery({ - conflicts: 'proceed', - index: conversationIndex, - query: { - ids: { - values: [existingConversation.id ?? ''], - }, - }, - refresh: false, - script: { - lang: 'painless', - params: { - ...params, - // when assigning undefined in painless, it will remove property and wil set it to null - // for patch we don't want to remove unspecified value in payload - assignEmpty: !(isPatch ?? true), + try { + const response = await esClient.updateByQuery({ + conflicts: 'proceed', + index: conversationIndex, + query: { + ids: { + values: [existingConversation.id ?? ''], + }, }, - source: ` + refresh: true, + script: { + lang: 'painless', + params: { + ...params, + // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !(isPatch ?? true), + }, + source: ` if (params.assignEmpty == true || params.containsKey('api_config')) { if (params.assignEmpty == true || params.api_config.containsKey('connector_id')) { ctx._source.api_config.connector_id = params.api_config.connector_id; @@ -119,29 +116,27 @@ export const updateConversation = async ( } ctx._source.updated_at = params.updated_at; `, - }, - }); + }, + }); - let updatedOCCVersion: string | undefined; - if (response.updated) { - const checkIfListUpdated = async (): Promise => { - const updatedConversation = await getConversation( - esClient, - conversationIndex, - existingConversation.id ?? '' + if (response.failures && response.failures.length > 0) { + logger.warn( + `Error updating conversation: ${response.failures.map((f) => f.id)} by ID: ${ + existingConversation.id + }` ); - /* if (updatedList?._version === list._version) { - throw Error('Document has not been re-indexed in time'); - } - updatedOCCVersion = updatedList?._version; - */ - }; - - await waitUntilDocumentIndexed(checkIfListUpdated); - } else { - throw Error('No conversation has been updated'); + return null; + } + } catch (err) { + logger.warn(`Error updating conversation: ${err} by ID: ${existingConversation.id}`); + throw err; } - return getConversation(esClient, conversationIndex, existingConversation.id ?? ''); + const updatedConversation = await getConversation( + esClient, + conversationIndex, + existingConversation.id ?? '' + ); + return updatedConversation; }; export const transformToUpdateScheme = ( @@ -153,7 +148,7 @@ export const transformToUpdateScheme = ( messages, replacements, }: ConversationUpdateProps -) => { +): UpdateConversationSchema => { return { updated_at: updatedAt, title, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/append_conversation_messages_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts new file mode 100644 index 0000000000000..eca60c5ca2db7 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { bulkActionConversationsRoute } from './bulk_actions_route'; +import { serverMock } from '../../__mocks__/server'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { getConversationsBulkActionRequest, requestMock } from '../../__mocks__/request'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION } from '@kbn/elastic-assistant-common'; +import { + getEmptyFindResult, + getFindConversationsResultWithSingleHit, +} from '../../__mocks__/response'; +import { getPerformBulkActionSchemaMock } from '../../__mocks__/conversations_schema.mock'; + +describe('Perform bulk action route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + let logger: ReturnType; + const mockConversation = getFindConversationsResultWithSingleHit().data[0]; + + beforeEach(() => { + server = serverMock.create(); + logger = loggingSystemMock.createLogger(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + bulkActionConversationsRoute(server.router, logger); + }); + + describe('status codes', () => { + it('returns 200 when performing bulk action with all dependencies present', async () => { + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + success: true, + rules_count: 1, + attributes: { + results: someBulkActionResults(), + summary: { + failed: 0, + skipped: 0, + succeeded: 1, + total: 1, + }, + }, + }); + }); + + it("returns 200 when provided filter query doesn't match any conversations", async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getEmptyFindResult() + ); + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + success: true, + rules_count: 0, + attributes: { + results: someBulkActionResults(), + summary: { + failed: 0, + skipped: 0, + succeeded: 0, + total: 0, + }, + }, + }); + }); + }); + + describe('rules execution failures', () => { + it('returns partial failure error if update of few rules fail', async () => { + ( + (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) + .bulk as jest.Mock + ).mockResolvedValue({ + rules: [mockConversation, mockConversation], + skipped: [], + errors: [ + { + message: 'mocked validation message', + conversation: { id: 'failed-conversation-id-1', name: 'Detect Root/Admin Users' }, + }, + { + message: 'mocked validation message', + conversation: { id: 'failed-conversation-id-2', name: 'Detect Root/Admin Users' }, + }, + { + message: 'test failure', + conversation: { id: 'failed-conversation-id-3', name: 'Detect Root/Admin Users' }, + }, + ], + total: 5, + }); + + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + attributes: { + summary: { + failed: 3, + succeeded: 2, + skipped: 0, + total: 5, + }, + errors: [ + { + message: 'mocked validation message', + conversations: [ + { + id: 'failed-rule-id-1', + name: 'Detect Root/Admin Users', + }, + { + id: 'failed-rule-id-2', + name: 'Detect Root/Admin Users', + }, + ], + status_code: 500, + }, + { + message: 'test failure', + rules: [ + { + id: 'failed-rule-id-3', + name: 'Detect Root/Admin Users', + }, + ], + status_code: 500, + }, + ], + results: someBulkActionResults(), + }, + message: 'Bulk edit partially failed', + status_code: 500, + }); + }); + }); + + describe('conversation skipping', () => { + it('returns partial failure error with skipped rules if some rule updates fail and others are skipped', async () => { + ( + (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) + .bulk as jest.Mock + ).mockResolvedValue({ + rules: [mockConversation, mockConversation], + skipped: [ + { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, + { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, + ], + errors: [ + { + message: 'test failure', + rule: { id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }, + }, + ], + total: 5, + }); + + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + attributes: { + summary: { + failed: 1, + skipped: 2, + succeeded: 2, + total: 5, + }, + errors: [ + { + message: 'test failure', + rules: [ + { + id: 'failed-rule-id-3', + name: 'Detect Root/Admin Users', + }, + ], + status_code: 500, + }, + ], + results: someBulkActionResults(), + }, + message: 'Bulk edit partially failed', + status_code: 500, + }); + }); + + it('returns success with skipped rules if some rules are skipped, but no errors are reported', async () => { + ( + (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) + .bulk as jest.Mock + ).mockResolvedValue({ + rules: [mockConversation, mockConversation], + skipped: [ + { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, + { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, + ], + errors: [], + total: 4, + }); + + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + attributes: { + summary: { + failed: 0, + skipped: 2, + succeeded: 2, + total: 4, + }, + results: someBulkActionResults(), + }, + rules_count: 4, + success: true, + }); + }); + + it('returns 500 with skipped rules if some rules are skipped, but some errors are reported', async () => { + ( + (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) + .bulk as jest.Mock + ).mockResolvedValue({ + rules: [mockConversation, mockConversation], + skipped: [ + { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, + { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, + ], + errors: [ + { + message: 'test failure', + rule: { id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }, + }, + ], + total: 5, + }); + + const response = await server.inject( + getConversationsBulkActionRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + attributes: { + summary: { + failed: 1, + skipped: 2, + succeeded: 2, + total: 5, + }, + results: someBulkActionResults(), + errors: [ + { + message: 'test failure', + rules: [{ id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }], + status_code: 500, + }, + ], + }, + message: 'Bulk edit partially failed', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + it('rejects payloads with no operations', async () => { + const request = requestMock.create({ + method: 'patch', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + body: { ...getPerformBulkActionSchemaMock(), action: undefined }, + }); + const result = server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + 'action: Invalid literal value, expected "delete", action: Invalid literal value, expected "disable", action: Invalid literal value, expected "enable", action: Invalid literal value, expected "export", action: Invalid literal value, expected "duplicate", and 2 more' + ); + }); + + it('accepts payloads with only delete action', async () => { + const request = requestMock.create({ + method: 'patch', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + body: getPerformBulkActionSchemaMock(), + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + it('accepts payloads with all operations', async () => { + const request = requestMock.create({ + method: 'patch', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + body: getPerformBulkActionSchemaMock(), + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + it('rejects payload if there is more than 100 updates in payload', async () => { + const request = requestMock.create({ + method: 'patch', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + body: { + ...getPerformBulkActionSchemaMock(), + ids: Array.from({ length: 101 }).map(() => 'fake-id'), + }, + }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(400); + expect(response.body.message).toEqual('More than 100 operations sent for bulk edit action.'); + }); + }); +}); + +function someBulkActionResults() { + return { + created: expect.any(Array), + deleted: expect.any(Array), + updated: expect.any(Array), + skipped: expect.any(Array), + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/bulk_actions_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts new file mode 100644 index 0000000000000..784cd48e6058e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { serverMock } from '../../__mocks__/server'; +import { createConversationRoute } from './create_route'; +import { + getBasicEmptySearchResponse, + getEmptyFindResult, + getFindConversationsResultWithSingleHit, +} from '../../__mocks__/response'; +import { getCreateConversationRequest, requestMock } from '../../__mocks__/request'; +import { + getCreateConversationSchemaMock, + getConversationMock, + getQueryConversationParams, +} from '../../__mocks__/conversations_schema.mock'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL } from '@kbn/elastic-assistant-common'; + +describe('Create conversation route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getEmptyFindResult() + ); // no current conversations + clients.elasticAssistant.getAIAssistantConversationsDataClient.createConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); // creation succeeds + + context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( + elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) + ); + createConversationRoute(server.router); + }); + + describe('status codes', () => { + test('returns 200 with a conversation created via AIAssistantConversationsDataClient', async () => { + const response = await server.inject( + getCreateConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + }); + + describe('unhappy paths', () => { + test('returns a duplicate error if conversation_id already exists', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + const response = await server.inject( + getCreateConversationRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(409); + expect(response.body).toEqual({ + message: expect.stringContaining('already exists'), + status_code: 409, + }); + }); + + test('catches error if creation throws', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.createConversation.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getCreateConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('allows rule type of query', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { + ...getCreateConversationSchemaMock(), + type: 'query', + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('disallows unknown rule type', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { + ...getCreateConversationSchemaMock(), + type: 'unexpected_type', + }, + }); + const result = server.validate(request); + + expect(result.badRequest).toHaveBeenCalled(); + }); + + test('allows rule type of query and custom from and interval', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { from: 'now-7m', interval: '5m', ...getCreateConversationSchemaMock() }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('disallows invalid "from" param on rule', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { + from: 'now-3755555555555555.67s', + interval: '5m', + ...getCreateConversationSchemaMock(), + }, + }); + const result = server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith('from: Failed to parse date-math expression'); + }); + }); + describe('rule containing response actions', () => { + beforeEach(() => { + // @ts-expect-error We're writting to a read only property just for the purpose of the test + clients.config.experimentalFeatures.endpointResponseActionsEnabled = true; + }); + const getResponseAction = (command: string = 'isolate') => ({ + action_type_id: '.endpoint', + params: { + command, + comment: '', + }, + }); + const defaultAction = getResponseAction(); + + test('is successful', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { + ...getCreateConversationSchemaMock(), + response_actions: [defaultAction], + }, + }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + expect(response.status).toEqual(200); + }); + + test('fails when provided with an unsupported command', async () => { + const wrongAction = getResponseAction('processes'); + + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + body: { + ...getCreateConversationSchemaMock(), + response_actions: [wrongAction], + }, + }); + const result = await server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + 'response_actions.0.action_type_id: Invalid literal value, expected ".osquery", response_actions.0.params.command: Invalid literal value, expected "isolate"' + ); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/create_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts new file mode 100644 index 0000000000000..0dec9a69e3de0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assistant-common'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { serverMock } from '../../__mocks__/server'; +import { deleteConversationRoute } from './delete_route'; +import { getDeleteConversationRequest, requestMock } from '../../__mocks__/request'; +import { + getEmptyFindResult, + getFindConversationsResultWithSingleHit, +} from '../../__mocks__/response'; + +describe('Delete conversation route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + deleteConversationRoute(server.router); + }); + + describe('status codes with getAIAssistantConversationsDataClient', () => { + test('returns 200 when deleting a single conversation with a valid getAIAssistantConversationsDataClient by Id', async () => { + const response = await server.inject( + getDeleteConversationRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(200); + }); + + test('returns 404 when deleting a single rule that does not exist with a valid actionClient and alertClient', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getEmptyFindResult() + ); + + const response = await server.inject( + getDeleteConversationRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(404); + expect(response.body).toEqual({ + message: 'conversation id: "conversation-1" not found', + status_code: 404, + }); + }); + + test('catches error if deletion throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.deleteConversation.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getDeleteConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('rejects a request with no id', async () => { + const request = requestMock.create({ + method: 'delete', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + query: {}, + }); + const response = await server.inject(request, requestContextMock.convertContext(context)); + expect(response.status).toEqual(400); + expect(response.body).toEqual({ + message: ['either "id" or "rule_id" must be set'], + status_code: 400, + }); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/delete_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts new file mode 100644 index 0000000000000..c2677a6173ccf --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { findConversationsRoute } from './find_route'; +import { getFindRequest, requestMock } from '../../__mocks__/request'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND } from '@kbn/elastic-assistant-common'; +import { serverMock } from '../../__mocks__/server'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; +import { + getConversationMock, + getQueryConversationParams, +} from '../../__mocks__/conversations_schema.mock'; + +describe('Find conversations route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + let logger: ReturnType; + + beforeEach(async () => { + server = serverMock.create(); + logger = loggingSystemMock.createLogger(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); + + findConversationsRoute(server.router, logger); + }); + + describe('status codes', () => { + test('returns 200', async () => { + const response = await server.inject( + getFindRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('catches error if search throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getFindRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('allows optional query params', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, + query: { + page: 2, + per_page: 20, + sort_field: 'name', + fields: ['field1', 'field2'], + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('ignores unknown query params', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, + query: { + invalid_value: 'test 1', + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/find_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts new file mode 100644 index 0000000000000..5b270e5ad9996 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { getFindRequest, requestMock } from '../../__mocks__/request'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS } from '@kbn/elastic-assistant-common'; +import { serverMock } from '../../__mocks__/server'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; +import { findUserConversationsRoute } from './find_user_conversations_route'; +import { + getConversationMock, + getQueryConversationParams, +} from '../../__mocks__/conversations_schema.mock'; + +describe('Find user conversations route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + let logger: ReturnType; + + beforeEach(async () => { + server = serverMock.create(); + logger = loggingSystemMock.createLogger(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); + + findUserConversationsRoute(server.router); + }); + + describe('status codes', () => { + test('returns 200', async () => { + const response = await server.inject( + getFindRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('catches error if search throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getFindRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('allows optional query params', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + query: { + page: 2, + per_page: 20, + sort_field: 'name', + fields: ['field1', 'field2'], + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('ignores unknown query params', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + query: { + invalid_value: 'test 1', + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/find_user_conversations_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts new file mode 100644 index 0000000000000..2fc645ad52c1b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { serverMock } from '../../__mocks__/server'; +import { readConversationRoute } from './read_route'; +import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; +import { + getConversationReadRequest, + getConversationReadRequestWithId, + requestMock, +} from '../../__mocks__/request'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assistant-common'; + +describe('Read conversation route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + let logger: ReturnType; + + const myFakeId = '99403909-ca9b-49ba-9d7a-7e5320e68d05'; + beforeEach(() => { + server = serverMock.create(); + logger = loggingSystemMock.createLogger(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getFindConversationsResultWithSingleHit() + ); + readConversationRoute(server.router); + }); + + describe('status codes', () => { + test('returns 200', async () => { + const response = await server.inject( + getConversationReadRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('returns 200 when reading a single rule outcome === exactMatch', async () => { + const response = await server.inject( + getConversationReadRequestWithId(myFakeId), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('catches error if search throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getConversationReadRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('data validation', () => { + test('returns 404 if given a non-existent id', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + {} + ); + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + query: { id: 'DNE_RULE' }, + }); + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(404); + expect(response.body).toEqual({ message: 'rule_id: "DNE_RULE" not found', status_code: 404 }); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/read_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts new file mode 100644 index 0000000000000..48e0139204459 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assistant-common'; +import { getUpdateConversationRequest, requestMock } from '../../__mocks__/request'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { serverMock } from '../../__mocks__/server'; +import { + getConversationMock, + getQueryConversationParams, + getUpdateConversationSchemaMock, +} from '../../__mocks__/conversations_schema.mock'; +import { getEmptyFindResult } from '../../__mocks__/response'; +import { updateConversationRoute } from './update_route'; + +describe('Update conversation route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getEmptyFindResult() + ); // no current conversations + clients.elasticAssistant.getAIAssistantConversationsDataClient.createConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); // creation succeeds + clients.elasticAssistant.getAIAssistantConversationsDataClient.updateConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); // successful update + + updateConversationRoute(server.router); + }); + + describe('status codes', () => { + test('returns 200', async () => { + const response = await server.inject( + getUpdateConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('returns 404 when updating a single rule that does not exist', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( + getEmptyFindResult() + ); + + const response = await server.inject( + getUpdateConversationRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(404); + expect(response.body).toEqual({ + message: 'rule_id: "rule-1" not found', + status_code: 404, + }); + }); + + test('catches error if search throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getUpdateConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('rejects payloads with no ID', async () => { + const noIdRequest = requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + body: { + ...getUpdateConversationSchemaMock(), + id: undefined, + }, + }); + const response = await server.inject(noIdRequest, requestContextMock.convertContext(context)); + expect(response.body).toEqual({ + message: ['either "id" or "rule_id" must be set'], + status_code: 400, + }); + }); + + test('allows query rule type', async () => { + const request = requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + body: { ...getUpdateConversationSchemaMock(), type: 'query' }, + }); + const result = await server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('rejects unknown rule type', async () => { + const request = requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + body: { ...getUpdateConversationSchemaMock(), type: 'unknown type' }, + }); + const result = await server.validate(request); + + expect(result.badRequest).toHaveBeenCalled(); + }); + + test('allows rule type of query and custom from and interval', async () => { + const request = requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + body: { + from: 'now-7m', + interval: '5m', + ...getUpdateConversationSchemaMock(), + type: 'query', + }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('disallows invalid "from" param on rule', async () => { + const request = requestMock.create({ + method: 'put', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + body: { + from: 'now-3755555555555555.67s', + interval: '5m', + ...getUpdateConversationSchemaMock(), + type: 'query', + }, + }); + const result = server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith('from: Failed to parse date-math expression'); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/routes/conversation/update_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index 82a9153fa50ea..bcb5de73e3c9a 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -13,14 +13,14 @@ import { ElasticAssistantPluginSetupDependencies, GetElser, } from '../types'; -import { createConversationRoute } from './conversation/create_route'; -import { deleteConversationRoute } from './conversation/delete_route'; -import { findConversationsRoute } from './conversation/find_route'; -import { readConversationRoute } from './conversation/read_route'; -import { updateConversationRoute } from './conversation/update_route'; -import { findUserConversationsRoute } from './conversation/find_user_conversations_route'; -import { bulkActionConversationsRoute } from './conversation/bulk_actions_route'; -import { appendConversationMessageRoute } from './conversation/append_conversation_messages_route'; +import { createConversationRoute } from './conversations/create_route'; +import { deleteConversationRoute } from './conversations/delete_route'; +import { findConversationsRoute } from './conversations/find_route'; +import { readConversationRoute } from './conversations/read_route'; +import { updateConversationRoute } from './conversations/update_route'; +import { findUserConversationsRoute } from './conversations/find_user_conversations_route'; +import { bulkActionConversationsRoute } from './conversations/bulk_actions_route'; +import { appendConversationMessageRoute } from './conversations/append_conversation_messages_route'; import { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; import { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index d8cfc46ec5a22..7f3d38580b1ad 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -49,14 +49,20 @@ export const getComments = ({ regenerateMessage, showAnonymizedValues, }: { - amendMessage: ({ conversationId, content }: { conversationId: string; content: string }) => void; + amendMessage: ({ + conversationId, + content, + }: { + conversationId: string; + content: string; + }) => Promise; currentConversation: Conversation; isFetchingResponse: boolean; regenerateMessage: (conversationId: string) => void; showAnonymizedValues: boolean; }): EuiCommentProps[] => { - const amendMessageOfConversation = (content: string) => { - amendMessage({ + const amendMessageOfConversation = async (content: string) => { + await amendMessage({ conversationId: currentConversation.id, content, }); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx index bc1e20bba2a77..5cc107068ae19 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx @@ -15,7 +15,7 @@ import { MessagePanel } from './message_panel'; import { MessageText } from './message_text'; interface Props { - amendMessage: (message: string) => void; + amendMessage: (message: string) => Promise; content?: string; isError?: boolean; isFetching?: boolean; diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_assistant_provider.tsx b/x-pack/plugins/security_solution/public/common/mock/mock_assistant_provider.tsx index 4dc5f01b0ee7d..d05bcc15bc107 100644 --- a/x-pack/plugins/security_solution/public/common/mock/mock_assistant_provider.tsx +++ b/x-pack/plugins/security_solution/public/common/mock/mock_assistant_provider.tsx @@ -10,6 +10,7 @@ import { actionTypeRegistryMock } from '@kbn/triggers-actions-ui-plugin/public/a import React from 'react'; import type { AssistantAvailability } from '@kbn/elastic-assistant'; import { AssistantProvider } from '@kbn/elastic-assistant'; +import { BASE_SECURITY_CONVERSATIONS } from '../../assistant/content/conversations'; interface Props { children: React.ReactNode; @@ -45,11 +46,10 @@ export const MockAssistantProviderComponent: React.FC = ({ children }) => }} defaultAllowReplacement={[]} getComments={jest.fn(() => [])} - getInitialConversations={jest.fn(() => ({}))} - setConversations={jest.fn()} setDefaultAllow={jest.fn()} setDefaultAllowReplacement={jest.fn()} http={mockHttp} + baseConversations={BASE_SECURITY_CONVERSATIONS} > {children} diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/rule_execution_status/rule_status_failed_callout.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/rule_execution_status/rule_status_failed_callout.test.tsx index a1f7b488b1d0a..3f8b9ef5386b2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/rule_execution_status/rule_status_failed_callout.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/rule_execution_status/rule_status_failed_callout.test.tsx @@ -16,6 +16,7 @@ import type { AssistantAvailability } from '@kbn/elastic-assistant'; import { httpServiceMock } from '@kbn/core-http-browser-mocks'; import { actionTypeRegistryMock } from '@kbn/triggers-actions-ui-plugin/public/application/action_type_registry.mock'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { BASE_SECURITY_CONVERSATIONS } from '../../../../assistant/content/conversations'; jest.mock('../../../../common/lib/kibana'); @@ -24,7 +25,6 @@ const DATE = '2022-01-27T15:03:31.176Z'; const MESSAGE = 'This rule is attempting to query data but...'; const actionTypeRegistry = actionTypeRegistryMock.create(); -const mockGetInitialConversations = jest.fn(() => ({})); const mockGetComments = jest.fn(() => []); const mockHttp = httpServiceMock.createStartContract({ basePath: '/test' }); const mockAssistantAvailability: AssistantAvailability = { @@ -61,10 +61,9 @@ const ContextWrapper: React.FC = ({ children }) => ( ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', DOC_LINK_VERSION: 'current', }} - getInitialConversations={mockGetInitialConversations} getComments={mockGetComments} http={mockHttp} - setConversations={jest.fn()} + baseConversations={BASE_SECURITY_CONVERSATIONS} setDefaultAllow={jest.fn()} setDefaultAllowReplacement={jest.fn()} > From a4e97b3b7ce38414b45050b62b09e85e0443c2ef Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Mon, 22 Jan 2024 13:44:04 -0800 Subject: [PATCH 027/141] cleanup --- .../server/lib/wait_until_document_indexed.ts | 25 ------------------- .../timeline/tabs_content/index.tsx | 4 +-- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts diff --git a/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts b/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts deleted file mode 100644 index a00ce684b5cb7..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/wait_until_document_indexed.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import pRetry from 'p-retry'; - -// index.refresh_interval -// https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings -const DEFAULT_INDEX_REFRESH_TIME = 1000; - -/** - * retries until item has been re-indexed - * Data stream and using update_by_query, delete_by_query which do support only refresh=true/false, - * this utility needed response back when updates/delete applied - * @param fn execution function to retry - */ -export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise => { - await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); - await pRetry(fn, { - minTimeout: DEFAULT_INDEX_REFRESH_TIME, - retries: 5, - }); -}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index 0c9282da70aef..9ae13a8364f69 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -135,7 +135,7 @@ const ActiveTimelineTab = memo( [activeTimelineTab] ); - const memoAssTab = useCallback(() => { + const getAssistantTab = useCallback(() => { if (showTimeline) { const AssistantTab = tabWithSuspense(lazy(() => import('../assistant_tab_content'))); return ( @@ -216,7 +216,7 @@ const ActiveTimelineTab = memo( > {isGraphOrNotesTabs && getTab(activeTimelineTab)} - {hasAssistantPrivilege ? memoAssTab() : null} + {hasAssistantPrivilege ? getAssistantTab() : null} ); } From 1dcb3186f89eb764117a8985480717a158bbb26a Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Mon, 22 Jan 2024 19:21:41 -0800 Subject: [PATCH 028/141] fixed tests --- .../assistant/assistant_header/index.test.tsx | 18 +- .../chat_send/use_chat_send.test.tsx | 2 + .../assistant/chat_send/use_chat_send.tsx | 2 +- .../conversation_selector/index.test.tsx | 170 +++++---------- .../conversation_selector_settings/index.tsx | 2 + .../system_prompt_settings.test.tsx | 8 +- .../use_perform_evaluation.test.tsx | 2 + .../assistant/use_conversation/index.test.tsx | 206 ++---------------- .../impl/assistant_context/index.test.tsx | 12 +- .../use_delete_knowledge_base.test.tsx | 4 + .../use_knowledge_base_status.test.tsx | 4 + .../use_setup_knowledge_base.test.tsx | 2 + .../delete_conversation.ts | 14 +- 13 files changed, 114 insertions(+), 332 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx index a4c0e2fe20719..d1ab1605f4f30 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx @@ -24,8 +24,11 @@ const testProps = { onToggleShowAnonymizedValues: jest.fn(), selectedConversationId: emptyWelcomeConvo.id, setIsSettingsModalVisible: jest.fn(), - setSelectedConversationId: jest.fn(), + setCurrentConversation: jest.fn(), + onConversationDeleted: jest.fn(), showAnonymizedValues: false, + conversations: {}, + refetchConversationsState: jest.fn(), }; describe('AssistantHeader', () => { @@ -53,11 +56,7 @@ describe('AssistantHeader', () => { it('showAnonymizedValues is not checked when currentConversation.replacements has values and showAnonymizedValues is false', () => { const { getByTestId } = render( - , + , { wrapper: TestProviders, } @@ -67,12 +66,7 @@ describe('AssistantHeader', () => { it('showAnonymizedValues is checked when currentConversation.replacements has values and showAnonymizedValues is true', () => { const { getByTestId } = render( - , + , { wrapper: TestProviders, } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index da03b876813bc..4fb3f6bf9bde4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -27,6 +27,7 @@ const appendMessage = jest.fn(); const removeLastMessage = jest.fn(); const appendReplacements = jest.fn(); const clearConversation = jest.fn(); +const refresh = jest.fn(); export const testProps: UseChatSendProps = { selectedPromptContexts: {}, @@ -45,6 +46,7 @@ export const testProps: UseChatSendProps = { setPromptTextPreview, setSelectedPromptContexts, setUserPrompt, + refresh, }; const robotMessage = { response: 'Response message from the robot', isError: false }; describe('use chat send', () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 86bda9b22a222..ab14aa845853c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -175,8 +175,8 @@ export const useChatSend = ({ setPromptTextPreview(''); setUserPrompt(''); setSelectedPromptContexts({}); - await clearConversation(currentConversation.id); setEditingSystemPromptId(defaultSystemPromptId); + await clearConversation(currentConversation.id); await refresh(); }, [ allSystemPrompts, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx index 4ab45c10b021a..b45f3391866be 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx @@ -25,17 +25,31 @@ const mockConversation = { setConversation, }; +const mockConversations = { + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, +}; + +const mockConversationsWithCustom = { + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + [customConvo.id]: customConvo, +}; + jest.mock('../../use_conversation', () => ({ useConversation: () => mockConversation, })); const onConversationSelected = jest.fn(); +const onConversationDeleted = jest.fn(); const defaultProps = { isDisabled: false, onConversationSelected, selectedConversationId: 'Welcome', defaultConnectorId: '123', defaultProvider: OpenAiProviderType.OpenAi, + conversations: mockConversations, + onConversationDeleted, }; describe('Conversation selector', () => { beforeAll(() => { @@ -46,14 +60,7 @@ describe('Conversation selector', () => { }); it('renders with correct selected conversation', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > + ); @@ -62,31 +69,17 @@ describe('Conversation selector', () => { }); it('On change, selects new item', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > + ); fireEvent.click(getByTestId('comboBoxSearchInput')); fireEvent.click(getByTestId(`convo-option-${alertConvo.id}`)); - expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id); + expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id, alertConvo.title); }); it('On clear input, clears selected options', () => { const { getByPlaceholderText, queryByPlaceholderText, getByTestId, queryByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > + ); @@ -99,14 +92,7 @@ describe('Conversation selector', () => { it('We can add a custom option', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > + ); @@ -117,31 +103,15 @@ describe('Conversation selector', () => { code: 'Enter', charCode: 13, }); - expect(setConversation).toHaveBeenCalledWith({ - conversation: { - id: customOption, - messages: [], - apiConfig: { - connectorId: '123', - defaultSystemPromptId: undefined, - provider: 'OpenAI', - }, - }, - }); + expect(onConversationSelected).toHaveBeenCalledWith(customOption, customOption); }); it('Only custom options can be deleted', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -156,16 +126,10 @@ describe('Conversation selector', () => { it('Custom options can be deleted', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -176,21 +140,16 @@ describe('Conversation selector', () => { jest.runAllTimers(); expect(onConversationSelected).not.toHaveBeenCalled(); - expect(deleteConversation).toHaveBeenCalledWith(customConvo.id); + expect(onConversationDeleted).toHaveBeenCalledWith(customConvo.id); }); it('Previous conversation is set to active when selected conversation is deleted', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -198,21 +157,15 @@ describe('Conversation selector', () => { fireEvent.click( within(getByTestId(`convo-option-${customConvo.id}`)).getByTestId('delete-option') ); - expect(onConversationSelected).toHaveBeenCalledWith(welcomeConvo.id); + expect(onConversationSelected).toHaveBeenCalledWith(welcomeConvo.id, welcomeConvo.title); }); it('Left arrow selects first conversation', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -222,21 +175,15 @@ describe('Conversation selector', () => { code: 'ArrowLeft', charCode: 27, }); - expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id); + expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id, alertConvo.title); }); it('Right arrow selects last conversation', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -246,21 +193,15 @@ describe('Conversation selector', () => { code: 'ArrowRight', charCode: 26, }); - expect(onConversationSelected).toHaveBeenCalledWith(customConvo.id); + expect(onConversationSelected).toHaveBeenCalledWith(customConvo.id, customConvo.title); }); it('Right arrow does nothing when ctrlKey is false', () => { const { getByTestId } = render( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, - }), - }} - > - + + ); @@ -275,14 +216,13 @@ describe('Conversation selector', () => { it('Right arrow does nothing when conversation lenth is 1', () => { const { getByTestId } = render( - ({ + + - + }} + /> ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index 1ba589102fcb4..fb4ef453c8d2a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -275,6 +275,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( = React.memo( { return fn(mockSystemPrompts); }); -const setUpdatedConversationSettings = jest.fn().mockImplementation((fn) => { +const setConversationSettings = jest.fn().mockImplementation((fn) => { return fn({ [welcomeConvo.id]: welcomeConvo, [alertConvo.id]: alertConvo, @@ -31,8 +31,10 @@ const testProps = { onSelectedSystemPromptChange, selectedSystemPrompt: mockSystemPrompts[0], setUpdatedSystemPromptSettings, - setUpdatedConversationSettings, + setConversationSettings, systemPromptSettings: mockSystemPrompts, + conversationsSettingsBulkActions: {}, + setConversationsSettingsBulkActions: jest.fn(), }; jest.mock('./system_prompt_selector/system_prompt_selector', () => ({ @@ -126,7 +128,7 @@ describe('SystemPromptSettings', () => { ); fireEvent.click(getByTestId('change-multi')); - expect(setUpdatedConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveReturnedWith({ [welcomeConvo.id]: { ...welcomeConvo, apiConfig: { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx index b065338480549..67e9c16ee1009 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx @@ -67,6 +67,7 @@ describe('usePerformEvaluation', () => { outputIndex: undefined, }, signal: undefined, + version: '1', }); expect(toasts.addError).not.toHaveBeenCalled(); }); @@ -106,6 +107,7 @@ describe('usePerformEvaluation', () => { outputIndex: 'outputIndex', }, signal: undefined, + version: '1', }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index be94a164364aa..dbe2e6979eeb1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -11,6 +11,7 @@ import { TestProviders } from '../../mock/test_providers/test_providers'; import { alertConvo, welcomeConvo } from '../../mock/conversation'; import React from 'react'; import { ConversationRole } from '../../assistant_context/types'; +import { updateConversationApi } from '../api'; const message = { content: 'You are a robot', role: 'user' as ConversationRole, @@ -24,15 +25,9 @@ const anotherMessage = { const mockConvo = { id: 'new-convo', + title: 'new-convo', messages: [message, anotherMessage], apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, - theme: { - title: 'Elastic AI Assistant', - titleIcon: 'logoSecurity', - assistant: { name: 'Assistant', icon: 'logoSecurity' }, - system: { icon: 'logoElastic' }, - user: {}, - }, }; describe('useConversation', () => { @@ -42,27 +37,16 @@ describe('useConversation', () => { it('should append a message to an existing conversation when called with valid conversationId and message', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - const appendResult = result.current.appendMessage({ + const appendResult = await result.current.appendMessage({ conversationId: welcomeConvo.id, message, }); expect(appendResult).toHaveLength(3); - expect(appendResult[2]).toEqual(message); + expect(appendResult![2]).toEqual(message); }); }); @@ -73,10 +57,6 @@ describe('useConversation', () => { wrapper: ({ children }) => ( ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), assistantTelemetry: { reportAssistantInvoked: () => {}, reportAssistantQuickPrompt: () => {}, @@ -90,7 +70,7 @@ describe('useConversation', () => { ), }); await waitForNextUpdate(); - result.current.appendMessage({ + await result.current.appendMessage({ conversationId: welcomeConvo.id, message, }); @@ -106,24 +86,15 @@ describe('useConversation', () => { it('should create a new conversation when called with valid conversationId and message', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - const createResult = result.current.createConversation({ - conversationId: mockConvo.id, + const createResult = await result.current.createConversation({ + id: mockConvo.id, messages: mockConvo.messages, + apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, + title: mockConvo.title, }); expect(createResult).toEqual(mockConvo); @@ -133,23 +104,11 @@ describe('useConversation', () => { it('should delete an existing conversation when called with valid conversationId', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [mockConvo.id]: mockConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - const deleteResult = result.current.deleteConversation('new-convo'); + const deleteResult = await result.current.deleteConversation('new-convo'); expect(deleteResult).toEqual(mockConvo); }); @@ -159,24 +118,14 @@ describe('useConversation', () => { await act(async () => { const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [welcomeConvo.id]: welcomeConvo, - }), - setConversations, - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - result.current.setApiConfig({ + await result.current.setApiConfig({ conversationId: welcomeConvo.id, apiConfig: mockConvo.apiConfig, + title: welcomeConvo.title, }); expect(setConversations).toHaveBeenCalledWith({ @@ -185,98 +134,15 @@ describe('useConversation', () => { }); }); - it('overwrites a conversation', async () => { - await act(async () => { - const setConversations = jest.fn(); - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - setConversations, - }} - > - {children} - - ), - }); - await waitForNextUpdate(); - - result.current.setConversation({ - conversation: { - ...mockConvo, - id: welcomeConvo.id, - }, - }); - - expect(setConversations).toHaveBeenCalledWith({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: { ...mockConvo, id: welcomeConvo.id }, - }); - }); - }); - - it('clears a conversation', async () => { - await act(async () => { - const setConversations = jest.fn(); - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - setConversations, - }} - > - {children} - - ), - }); - await waitForNextUpdate(); - - result.current.clearConversation(welcomeConvo.id); - - expect(setConversations).toHaveBeenCalledWith({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: { - ...welcomeConvo, - apiConfig: { - ...welcomeConvo.apiConfig, - defaultSystemPromptId: 'default-system-prompt', - }, - messages: [], - replacements: undefined, - }, - }); - }); - }); - it('appends replacements', async () => { await act(async () => { const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - setConversations, - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - result.current.appendReplacements({ + await result.current.appendReplacements({ conversationId: welcomeConvo.id, replacements: { '1.0.0.721': '127.0.0.1', @@ -285,7 +151,7 @@ describe('useConversation', () => { }, }); - expect(setConversations).toHaveBeenCalledWith({ + expect(updateConversationApi).toHaveBeenCalledWith({ [alertConvo.id]: alertConvo, [welcomeConvo.id]: { ...welcomeConvo, @@ -303,24 +169,11 @@ describe('useConversation', () => { await act(async () => { const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [mockConvo.id]: mockConvo, - }), - setConversations, - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - const removeResult = result.current.removeLastMessage('new-convo'); + const removeResult = await result.current.removeLastMessage('new-convo'); expect(removeResult).toEqual([message]); expect(setConversations).toHaveBeenCalledWith({ @@ -335,24 +188,11 @@ describe('useConversation', () => { await act(async () => { const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [mockConvo.id]: mockConvo, - }), - setConversations, - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); - result.current.amendMessage({ + await result.current.amendMessage({ conversationId: 'new-convo', content: 'hello world', }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx index a8dc5b1aa1db7..ccfc53cbaacb6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx @@ -35,22 +35,22 @@ describe('AssistantContext', () => { expect(result.current.http.fetch).toBeCalledWith(path); }); - test('getConversationId defaults to provided id', async () => { + test('getLastConversationId defaults to provided id', async () => { const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getConversationId('123'); + const id = result.current.getLastConversationId('123'); expect(id).toEqual('123'); }); - test('getConversationId uses local storage id when no id is provided ', async () => { + test('getLastConversationId uses local storage id when no id is provided ', async () => { const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getConversationId(); + const id = result.current.getLastConversationId(); expect(id).toEqual('456'); }); - test('getConversationId defaults to Welcome when no local storage id and no id is provided ', async () => { + test('getLastConversationId defaults to Welcome when no local storage id and no id is provided ', async () => { (useLocalStorage as jest.Mock).mockReturnValue([undefined, jest.fn()]); const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getConversationId(); + const id = result.current.getLastConversationId(); expect(id).toEqual('Welcome'); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_delete_knowledge_base.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_delete_knowledge_base.test.tsx index 4b2061447859d..06933c4ebeff9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_delete_knowledge_base.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_delete_knowledge_base.test.tsx @@ -58,6 +58,8 @@ describe('useDeleteKnowledgeBase', () => { '/internal/elastic_assistant/knowledge_base/', { method: 'DELETE', + signal: undefined, + version: '1', } ); expect(toasts.addError).not.toHaveBeenCalled(); @@ -80,6 +82,8 @@ describe('useDeleteKnowledgeBase', () => { '/internal/elastic_assistant/knowledge_base/something', { method: 'DELETE', + signal: undefined, + version: '1', } ); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_knowledge_base_status.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_knowledge_base_status.test.tsx index 0c1810aaf04f0..a999666845378 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_knowledge_base_status.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_knowledge_base_status.test.tsx @@ -57,6 +57,8 @@ describe('useKnowledgeBaseStatus', () => { '/internal/elastic_assistant/knowledge_base/', { method: 'GET', + signal: undefined, + version: '1', } ); expect(toasts.addError).not.toHaveBeenCalled(); @@ -73,6 +75,8 @@ describe('useKnowledgeBaseStatus', () => { '/internal/elastic_assistant/knowledge_base/something', { method: 'GET', + signal: undefined, + version: '1', } ); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_setup_knowledge_base.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_setup_knowledge_base.test.tsx index 2a11fb368981d..8e0b084e9beed 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_setup_knowledge_base.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/use_setup_knowledge_base.test.tsx @@ -57,6 +57,7 @@ describe('useSetupKnowledgeBase', () => { '/internal/elastic_assistant/knowledge_base/', { method: 'POST', + version: '1', } ); expect(toasts.addError).not.toHaveBeenCalled(); @@ -79,6 +80,7 @@ describe('useSetupKnowledgeBase', () => { '/internal/elastic_assistant/knowledge_base/something', { method: 'POST', + version: '1', } ); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index 8f5aae3cc6af8..eb66807841821 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -6,7 +6,6 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; -import { waitUntilDocumentIndexed } from '../lib/wait_until_document_indexed'; import { getConversation } from './get_conversation'; export const deleteConversation = async ( @@ -26,19 +25,10 @@ export const deleteConversation = async ( }, conflicts: 'proceed', index: conversationIndex, - refresh: false, + refresh: true, }); - if (response.deleted) { - const checkIfConversationDeleted = async (): Promise => { - const deletedConversation = await getConversation(esClient, conversationIndex, id); - if (deletedConversation !== null) { - throw Error('Conversation has not been re-indexed in time'); - } - }; - - await waitUntilDocumentIndexed(checkIfConversationDeleted); - } else { + if (!response.deleted && response.deleted === 0) { throw Error('No conversation has been deleted'); } } From def6ecd90c6a9b2c0b3e8ff77b1a72964e64106b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 23 Jan 2024 03:32:50 +0000 Subject: [PATCH 029/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/packages/kbn-elastic-assistant/tsconfig.json | 1 + x-pack/plugins/elastic_assistant/tsconfig.json | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/kbn-elastic-assistant/tsconfig.json b/x-pack/packages/kbn-elastic-assistant/tsconfig.json index eea97cfc917dc..f168c27aa3618 100644 --- a/x-pack/packages/kbn-elastic-assistant/tsconfig.json +++ b/x-pack/packages/kbn-elastic-assistant/tsconfig.json @@ -30,5 +30,6 @@ "@kbn/ui-theme", "@kbn/core-doc-links-browser", "@kbn/core", + "@kbn/kibana-react-plugin", ] } diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index b1a401d8cfcd1..89394105d24b7 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -18,7 +18,6 @@ "@kbn/core-http-server", "@kbn/licensing-plugin", "@kbn/securitysolution-es-utils", - "@kbn/securitysolution-io-ts-utils", "@kbn/actions-plugin", "@kbn/elastic-assistant", "@kbn/logging-mocks", @@ -36,6 +35,18 @@ "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", "@kbn/data-stream-adapter", + "@kbn/alerts-as-data-utils", + "@kbn/core-saved-objects-utils-server", + "@kbn/core-elasticsearch-client-server-mocks", + "@kbn/task-manager-plugin", + "@kbn/security-plugin", + "@kbn/securitysolution-io-ts-list-types", + "@kbn/es-query", + "@kbn/es-types", + "@kbn/config-schema", + "@kbn/zod-helpers", + "@kbn/core-saved-objects-server", + "@kbn/spaces-plugin", ], "exclude": [ "target/**/*", From ed693974941eba1c3d1d2a079fc91c245789c436 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 23 Jan 2024 03:39:21 +0000 Subject: [PATCH 030/141] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d9449dd6b4105..8e5081d7b6138 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -318,6 +318,7 @@ src/plugins/dashboard @elastic/kibana-presentation src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery +packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore src/plugins/data_view_editor @elastic/kibana-data-discovery examples/data_view_field_editor_example @elastic/kibana-data-discovery src/plugins/data_view_field_editor @elastic/kibana-data-discovery From c953777eb6d742ec6e0961c4173c8e9f8ec5dc98 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Mon, 22 Jan 2024 20:01:18 -0800 Subject: [PATCH 031/141] - --- .../impl/assistant/api/index.test.tsx | 9 +++++++++ .../conversation_settings.test.tsx | 18 ++++++++++-------- .../mock/test_providers/test_providers.tsx | 6 +----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx index 328f5568cf628..79a7cb91c37de 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx @@ -59,6 +59,7 @@ describe('API tests', () => { headers: { 'Content-Type': 'application/json' }, method: 'POST', signal: undefined, + version: '1', } ); }); @@ -79,6 +80,7 @@ describe('API tests', () => { asResponse: true, rawResponse: true, signal: undefined, + version: '1', } ); }); @@ -105,6 +107,7 @@ describe('API tests', () => { }, method: 'POST', signal: undefined, + version: '1', } ); }); @@ -127,6 +130,7 @@ describe('API tests', () => { 'Content-Type': 'application/json', }, signal: undefined, + version: '1', } ); }); @@ -149,6 +153,7 @@ describe('API tests', () => { 'Content-Type': 'application/json', }, signal: undefined, + version: '1', } ); }); @@ -282,6 +287,7 @@ describe('API tests', () => { { method: 'GET', signal: undefined, + version: '1', } ); }); @@ -306,6 +312,7 @@ describe('API tests', () => { { method: 'POST', signal: undefined, + version: '1', } ); }); @@ -328,6 +335,7 @@ describe('API tests', () => { { method: 'DELETE', signal: undefined, + version: '1', } ); }); @@ -377,6 +385,7 @@ describe('API tests', () => { runName: 'Test Run Name', }, signal: undefined, + version: '1', }); }); it('returns error when error is an error', async () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx index 25e2f757bea5b..f6710fab23b6f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx @@ -21,7 +21,7 @@ const mockConvos = { }; const onSelectedConversationChange = jest.fn(); -const setUpdatedConversationSettings = jest.fn().mockImplementation((fn) => { +const setConversationSettings = jest.fn().mockImplementation((fn) => { return fn(mockConvos); }); @@ -33,7 +33,9 @@ const testProps = { http: { basePath: { get: jest.fn() } }, onSelectedConversationChange, selectedConversation: welcomeConvo, - setUpdatedConversationSettings, + setConversationSettings, + conversationsSettingsBulkActions: {}, + setConversationsSettingsBulkActions: jest.fn(), } as unknown as ConversationSettingsProps; jest.mock('../../../connectorland/use_load_connectors', () => ({ @@ -115,7 +117,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-sp')); - expect(setUpdatedConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveReturnedWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, @@ -135,7 +137,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-convo')); - expect(setUpdatedConversationSettings).toHaveReturnedWith(mockConvos); + expect(setConversationSettings).toHaveReturnedWith(mockConvos); expect(onSelectedConversationChange).toHaveBeenCalledWith(alertConvo); }); it('Selecting an existing conversation updates the selected convo and is added to the convo settings', () => { @@ -154,7 +156,7 @@ describe('ConversationSettings', () => { id: 'Cool new conversation', messages: [], }; - expect(setUpdatedConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveReturnedWith({ ...mockConvos, [newConvo.id]: newConvo, }); @@ -168,7 +170,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('delete-convo')); const { [customConvo.id]: _, ...rest } = mockConvos; - expect(setUpdatedConversationSettings).toHaveReturnedWith(rest); + expect(setConversationSettings).toHaveReturnedWith(rest); }); it('Selecting a new connector updates the conversation', () => { const { getByTestId } = render( @@ -177,7 +179,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-connector')); - expect(setUpdatedConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveReturnedWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, @@ -194,7 +196,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-model')); - expect(setUpdatedConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveReturnedWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx b/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx index 024e39ac46c3c..cacc6fe477ffe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx @@ -27,8 +27,6 @@ interface Props { window.scrollTo = jest.fn(); window.HTMLElement.prototype.scrollIntoView = jest.fn(); -const mockGetInitialConversations = () => ({}); - export const mockAssistantAvailability: AssistantAvailability = { hasAssistantPrivilege: false, hasConnectorsAllPrivilege: true, @@ -40,7 +38,6 @@ export const mockAssistantAvailability: AssistantAvailability = { export const TestProvidersComponent: React.FC = ({ assistantAvailability = mockAssistantAvailability, children, - getInitialConversations = mockGetInitialConversations, providerContext, }) => { const actionTypeRegistry = actionTypeRegistryMock.create(); @@ -83,11 +80,10 @@ export const TestProvidersComponent: React.FC = ({ DOC_LINK_VERSION: 'current', }} getComments={mockGetComments} - getInitialConversations={getInitialConversations} - setConversations={jest.fn()} setDefaultAllow={jest.fn()} setDefaultAllowReplacement={jest.fn()} http={mockHttp} + baseConversations={{}} {...providerContext} > {children} From d1a93d5d39548d1f9c52468edbee0ffd714ce7a5 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 23 Jan 2024 13:25:32 -0800 Subject: [PATCH 032/141] more tests fixes --- .../conversation_selector/index.test.tsx | 10 +--- .../impl/assistant/index.test.tsx | 34 ++++++------ .../impl/assistant/index.tsx | 9 ++- .../use_settings_updater.test.tsx | 43 ++++++++++----- .../connector_setup/index.test.tsx | 55 +++++-------------- .../mock/test_providers/test_providers.tsx | 3 +- .../mock/test_providers/test_providers.tsx | 4 +- .../translations/translations/fr-FR.json | 7 --- .../translations/translations/zh-CN.json | 7 --- 9 files changed, 72 insertions(+), 100 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx index b45f3391866be..b13164e1570b5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx @@ -93,7 +93,7 @@ describe('Conversation selector', () => { it('We can add a custom option', () => { const { getByTestId } = render( - + ); const customOption = 'Custom option'; @@ -181,9 +181,7 @@ describe('Conversation selector', () => { it('Right arrow selects last conversation', () => { const { getByTestId } = render( - + ); @@ -199,9 +197,7 @@ describe('Conversation selector', () => { it('Right arrow does nothing when ctrlKey is false', () => { const { getByTestId } = render( - + ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index 09815198f592d..040082e3559e9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -9,7 +9,6 @@ import React from 'react'; import { act, fireEvent, render, screen } from '@testing-library/react'; import { Assistant } from '.'; -import { Conversation } from '../assistant_context/types'; import type { IHttpFetchError } from '@kbn/core/public'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public'; @@ -30,27 +29,28 @@ jest.mock('react-use'); jest.mock('./prompt_editor', () => ({ PromptEditor: jest.fn() })); jest.mock('./quick_prompts/quick_prompts', () => ({ QuickPrompts: jest.fn() })); +jest.mock('./api/conversations/use_fetch_current_user_conversations', () => ({ + useFetchCurrentUserConversations: jest.fn().mockResolvedValue(() => ({ + [WELCOME_CONVERSATION_TITLE]: { + id: WELCOME_CONVERSATION_TITLE, + title: WELCOME_CONVERSATION_TITLE, + messages: [], + apiConfig: {}, + }, + [MOCK_CONVERSATION_TITLE]: { + id: MOCK_CONVERSATION_TITLE, + title: MOCK_CONVERSATION_TITLE, + messages: [], + apiConfig: {}, + }, + })), +})); const MOCK_CONVERSATION_TITLE = 'electric sheep'; -const getInitialConversations = (): Record => ({ - [WELCOME_CONVERSATION_TITLE]: { - id: WELCOME_CONVERSATION_TITLE, - title: WELCOME_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - }, - [MOCK_CONVERSATION_TITLE]: { - id: MOCK_CONVERSATION_TITLE, - title: MOCK_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - }, -}); - const renderAssistant = (extraProps = {}, providerProps = {}) => render( - + ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 776dbad58bf23..8bd13f9d04a96 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -236,7 +236,14 @@ const AssistantComponent: React.FC = ({ if (!currentConversation.excludeFromLastConversationStorage) { setLastConversationId(currentConversation.id); } - }, [areConnectorsFetched, connectors?.length, currentConversation, setLastConversationId]); + }, [ + areConnectorsFetched, + connectors?.length, + conversationsData, + currentConversation, + isLoading, + setLastConversationId, + ]); const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ conversation: blockBotConversation, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx index af73fa31293b3..358f8cf36cd2e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx @@ -15,12 +15,17 @@ import { mockSuperheroSystemPrompt, mockSystemPrompt, } from '../../../mock/system_prompt'; +import { HttpSetup } from '@kbn/core/public'; const mockConversations = { [alertConvo.id]: alertConvo, [welcomeConvo.id]: welcomeConvo, }; +const mockHttp = { + fetch: jest.fn(), +} as unknown as HttpSetup; + const mockSystemPrompts: Prompt[] = [mockSystemPrompt]; const mockQuickPrompts: Prompt[] = [defaultSystemPrompt]; @@ -29,14 +34,12 @@ const initialDefaultAllowReplacement = ['replacement1']; const setAllQuickPromptsMock = jest.fn(); const setAllSystemPromptsMock = jest.fn(); -const setConversationsMock = jest.fn(); const setDefaultAllowMock = jest.fn(); const setDefaultAllowReplacementMock = jest.fn(); const setKnowledgeBaseMock = jest.fn(); const reportAssistantSettingToggled = jest.fn(); const mockValues = { assistantTelemetry: { reportAssistantSettingToggled }, - conversations: mockConversations, allSystemPrompts: mockSystemPrompts, allQuickPrompts: mockQuickPrompts, defaultAllow: initialDefaultAllow, @@ -46,12 +49,13 @@ const mockValues = { isEnabledKnowledgeBase: true, latestAlerts: DEFAULT_LATEST_ALERTS, }, + baseConversations: {}, setAllQuickPrompts: setAllQuickPromptsMock, - setConversations: setConversationsMock, setAllSystemPrompts: setAllSystemPromptsMock, setDefaultAllow: setDefaultAllowMock, setDefaultAllowReplacement: setDefaultAllowReplacementMock, setKnowledgeBase: setKnowledgeBaseMock, + http: mockHttp, }; const updatedValues = { @@ -81,10 +85,11 @@ describe('useSettingsUpdater', () => { }); it('should set all state variables to their initial values when resetSettings is called', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater()); + const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater(mockConversations)); await waitForNextUpdate(); const { - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, setUpdatedDefaultAllow, @@ -93,7 +98,8 @@ describe('useSettingsUpdater', () => { resetSettings, } = result.current; - setUpdatedConversationSettings(updatedValues.conversations); + setConversationSettings(updatedValues.conversations); + setConversationsSettingsBulkActions({}); setUpdatedQuickPromptSettings(updatedValues.allQuickPrompts); setUpdatedSystemPromptSettings(updatedValues.allSystemPrompts); setUpdatedDefaultAllow(updatedValues.defaultAllow); @@ -109,7 +115,7 @@ describe('useSettingsUpdater', () => { resetSettings(); - expect(result.current.conversationSettings).toEqual(mockValues.conversations); + expect(result.current.conversationSettings).toEqual(mockConversations); expect(result.current.quickPromptSettings).toEqual(mockValues.allQuickPrompts); expect(result.current.systemPromptSettings).toEqual(mockValues.allSystemPrompts); expect(result.current.defaultAllow).toEqual(mockValues.defaultAllow); @@ -120,10 +126,11 @@ describe('useSettingsUpdater', () => { it('should update all state variables to their updated values when saveSettings is called', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater()); + const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater(mockConversations)); await waitForNextUpdate(); const { - setUpdatedConversationSettings, + setConversationSettings, + setConversationsSettingsBulkActions, setUpdatedQuickPromptSettings, setUpdatedSystemPromptSettings, setUpdatedDefaultAllow, @@ -131,7 +138,8 @@ describe('useSettingsUpdater', () => { setUpdatedKnowledgeBaseSettings, } = result.current; - setUpdatedConversationSettings(updatedValues.conversations); + setConversationSettings(updatedValues.conversations); + setConversationsSettingsBulkActions({ delete: { ids: ['1'] } }); setUpdatedQuickPromptSettings(updatedValues.allQuickPrompts); setUpdatedSystemPromptSettings(updatedValues.allSystemPrompts); setUpdatedDefaultAllow(updatedValues.defaultAllow); @@ -140,9 +148,16 @@ describe('useSettingsUpdater', () => { result.current.saveSettings(); + expect(mockHttp.fetch).toHaveBeenCalledWith( + '/api/elastic_assistant/conversations/_bulk_action', + { + method: 'POST', + version: '2023-10-31', + body: '{"delete":{"ids":["1"]}}', + } + ); expect(setAllQuickPromptsMock).toHaveBeenCalledWith(updatedValues.allQuickPrompts); expect(setAllSystemPromptsMock).toHaveBeenCalledWith(updatedValues.allSystemPrompts); - expect(setConversationsMock).toHaveBeenCalledWith(updatedValues.conversations); expect(setDefaultAllowMock).toHaveBeenCalledWith(updatedValues.defaultAllow); expect(setDefaultAllowReplacementMock).toHaveBeenCalledWith( updatedValues.defaultAllowReplacement @@ -152,7 +167,7 @@ describe('useSettingsUpdater', () => { }); it('should track which toggles have been updated when saveSettings is called', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater()); + const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater(mockConversations)); await waitForNextUpdate(); const { setUpdatedKnowledgeBaseSettings } = result.current; @@ -167,7 +182,7 @@ describe('useSettingsUpdater', () => { }); it('should track only toggles that updated', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater()); + const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater(mockConversations)); await waitForNextUpdate(); const { setUpdatedKnowledgeBaseSettings } = result.current; @@ -183,7 +198,7 @@ describe('useSettingsUpdater', () => { }); it('if no toggles update, do not track anything', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater()); + const { result, waitForNextUpdate } = renderHook(() => useSettingsUpdater(mockConversations)); await waitForNextUpdate(); const { setUpdatedKnowledgeBaseSettings } = result.current; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx index 50726ebd7f6a8..4b1ab2ec88038 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx @@ -14,9 +14,15 @@ import { TestProviders } from '../../mock/test_providers/test_providers'; import { EuiCommentList } from '@elastic/eui'; const onSetupComplete = jest.fn(); +const setConversations = jest.fn(); const defaultProps = { conversation: welcomeConvo, onSetupComplete, + setConversations, + conversations: { + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + }, }; const newConnector = { actionTypeId: '.gen-ai', name: 'cool name' }; jest.mock('../add_connector_modal', () => ({ @@ -32,7 +38,6 @@ jest.mock('../add_connector_modal', () => ({ ), })); -const setConversation = jest.fn(); const setApiConfig = jest.fn(); const mockConversation = { appendMessage: jest.fn(), @@ -41,7 +46,6 @@ const mockConversation = { createConversation: jest.fn(), deleteConversation: jest.fn(), setApiConfig, - setConversation, }; jest.mock('../../assistant/use_conversation', () => ({ @@ -56,18 +60,7 @@ describe('useConnectorSetup', () => { it('should render comments and prompts', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConnectorSetup(defaultProps), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); expect( @@ -78,7 +71,7 @@ describe('useConnectorSetup', () => { timestamp: 'at: 7/17/2023, 1:00:36 PM', }, { - username: 'Elastic AI Assistant', + username: 'Assistant', timestamp: 'at: 7/17/2023, 1:00:40 PM', }, ]); @@ -89,18 +82,7 @@ describe('useConnectorSetup', () => { it('should set api config for each conversation when new connector is saved', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConnectorSetup(defaultProps), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); const { getByTestId, queryByTestId, rerender } = render(result.current.prompt, { @@ -112,7 +94,7 @@ describe('useConnectorSetup', () => { rerender(result.current.prompt); fireEvent.click(getByTestId('modal-mock')); - expect(setApiConfig).toHaveBeenCalledTimes(2); + expect(setApiConfig).toHaveBeenCalledTimes(1); }); }); @@ -147,21 +129,10 @@ describe('useConnectorSetup', () => { expect(queryByTestId('connectorButton')).not.toBeInTheDocument(); }); }); - it('should call onSetupComplete and setConversation when onHandleMessageStreamingComplete', async () => { + it('should call onSetupComplete and setConversations when onHandleMessageStreamingComplete', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConnectorSetup(defaultProps), { - wrapper: ({ children }) => ( - ({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }), - }} - > - {children} - - ), + wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); render(, { @@ -170,7 +141,7 @@ describe('useConnectorSetup', () => { expect(clearTimeout).toHaveBeenCalled(); expect(onSetupComplete).toHaveBeenCalled(); - expect(setConversation).toHaveBeenCalled(); + expect(setConversations).toHaveBeenCalled(); }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx b/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx index cacc6fe477ffe..d44d5a62f69cb 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/mock/test_providers/test_providers.tsx @@ -15,12 +15,11 @@ import { ThemeProvider } from 'styled-components'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { AssistantProvider, AssistantProviderProps } from '../../assistant_context'; -import { AssistantAvailability, Conversation } from '../../assistant_context/types'; +import { AssistantAvailability } from '../../assistant_context/types'; interface Props { assistantAvailability?: AssistantAvailability; children: React.ReactNode; - getInitialConversations?: () => Record; providerContext?: Partial; } diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/test_providers/test_providers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/test_providers/test_providers.tsx index 175380cc5169a..2b961df9377a8 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/test_providers/test_providers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/test_providers/test_providers.tsx @@ -27,7 +27,6 @@ window.scrollTo = jest.fn(); export const TestProvidersComponent: React.FC = ({ children, isILMAvailable = true }) => { const http = httpServiceMock.createSetupContract({ basePath: '/test' }); const actionTypeRegistry = actionTypeRegistryMock.create(); - const mockGetInitialConversations = jest.fn(() => ({})); const mockGetComments = jest.fn(() => []); const mockHttp = httpServiceMock.createStartContract({ basePath: '/test' }); const mockTelemetryEvents = { @@ -71,11 +70,10 @@ export const TestProvidersComponent: React.FC = ({ children, isILMAvailab DOC_LINK_VERSION: 'current', }} getComments={mockGetComments} - getInitialConversations={mockGetInitialConversations} - setConversations={jest.fn()} setDefaultAllow={jest.fn()} setDefaultAllowReplacement={jest.fn()} http={mockHttp} + baseConversations={{}} > Date: Tue, 23 Jan 2024 15:14:36 -0800 Subject: [PATCH 033/141] more test --- .../conversation_selector_settings/index.tsx | 68 ++------- .../conversation_settings.test.tsx | 17 ++- .../conversation_settings.tsx | 7 +- .../impl/assistant/helpers.test.ts | 11 +- .../impl/assistant/index.test.tsx | 136 +++++++++++++----- .../settings/assistant_settings.test.tsx | 24 ++-- .../assistant/settings/assistant_settings.tsx | 2 +- .../assistant_settings_button.test.tsx | 7 +- .../assistant/use_conversation/index.test.tsx | 31 +++- .../connector_selector_inline.test.tsx | 10 ++ .../translations/translations/ja-JP.json | 7 - 11 files changed, 191 insertions(+), 129 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index fb4ef453c8d2a..3cc929de3f980 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -18,12 +18,10 @@ import { import React, { useCallback, useMemo, useState } from 'react'; import { css } from '@emotion/react'; -import useEvent from 'react-use/lib/useEvent'; import { Conversation } from '../../../..'; import * as i18n from '../conversation_selector/translations'; import { SystemPromptSelectorOption } from '../../prompt_editor/system_prompt/system_prompt_modal/system_prompt_selector/system_prompt_selector'; -const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; interface Props { conversations: Record; onConversationDeleted: (conversationId: string) => void; @@ -160,40 +158,6 @@ export const ConversationSelectorSettings: React.FC = React.memo( handleSelectionChange(nextOption); }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); - // Register keyboard listener for quick conversation switching - const onKeyDown = useCallback( - (event: KeyboardEvent) => { - if (isDisabled || conversationIds.length <= 1) { - return; - } - - if ( - event.key === 'ArrowLeft' && - (isMac ? event.metaKey : event.ctrlKey) && - !shouldDisableKeyboardShortcut() - ) { - event.preventDefault(); - onLeftArrowClick(); - } - if ( - event.key === 'ArrowRight' && - (isMac ? event.metaKey : event.ctrlKey) && - !shouldDisableKeyboardShortcut() - ) { - event.preventDefault(); - onRightArrowClick(); - } - }, - [ - conversationIds.length, - isDisabled, - onLeftArrowClick, - onRightArrowClick, - shouldDisableKeyboardShortcut, - ] - ); - useEvent('keydown', onKeyDown); - const renderOption: ( option: ConversationSelectorSettingsOption, searchValue: string, @@ -272,26 +236,22 @@ export const ConversationSelectorSettings: React.FC = React.memo( compressed={true} isDisabled={isDisabled} prepend={ - - - + } append={ - - - + } /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx index f6710fab23b6f..687b469113e37 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx @@ -21,9 +21,7 @@ const mockConvos = { }; const onSelectedConversationChange = jest.fn(); -const setConversationSettings = jest.fn().mockImplementation((fn) => { - return fn(mockConvos); -}); +const setConversationSettings = jest.fn(); const testProps = { allSystemPrompts: mockSystemPrompts, @@ -117,7 +115,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-sp')); - expect(setConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveBeenLastCalledWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, @@ -137,7 +135,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-convo')); - expect(setConversationSettings).toHaveReturnedWith(mockConvos); + expect(setConversationSettings).toHaveBeenLastCalledWith(mockConvos); expect(onSelectedConversationChange).toHaveBeenCalledWith(alertConvo); }); it('Selecting an existing conversation updates the selected convo and is added to the convo settings', () => { @@ -154,9 +152,10 @@ describe('ConversationSettings', () => { provider: 'OpenAI', }, id: 'Cool new conversation', + title: 'Cool new conversation', messages: [], }; - expect(setConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveBeenLastCalledWith({ ...mockConvos, [newConvo.id]: newConvo, }); @@ -170,7 +169,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('delete-convo')); const { [customConvo.id]: _, ...rest } = mockConvos; - expect(setConversationSettings).toHaveReturnedWith(rest); + expect(setConversationSettings).toHaveBeenLastCalledWith(rest); }); it('Selecting a new connector updates the conversation', () => { const { getByTestId } = render( @@ -179,7 +178,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-connector')); - expect(setConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveBeenLastCalledWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, @@ -196,7 +195,7 @@ describe('ConversationSettings', () => { ); fireEvent.click(getByTestId('change-model')); - expect(setConversationSettings).toHaveReturnedWith({ + expect(setConversationSettings).toHaveBeenLastCalledWith({ ...mockConvos, [welcomeConvo.id]: { ...welcomeConvo, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index 2d481b9c8abe5..8374dfe1d3f9a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -224,10 +224,10 @@ export const ConversationSettings: React.FC = React.m model: config?.defaultModel, }, }; - setConversationSettings((prev) => ({ - ...prev, + setConversationSettings({ + ...conversationSettings, [selectedConversation.id]: updatedConversation, - })); + }); if (selectedConversation.id !== selectedConversation.title) { setConversationsSettingsBulkActions({ ...conversationsSettingsBulkActions, @@ -261,6 +261,7 @@ export const ConversationSettings: React.FC = React.m } }, [ + conversationSettings, conversationsSettingsBulkActions, selectedConversation, setConversationSettings, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts index b176a229bcca7..331361679ba75 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts @@ -24,6 +24,7 @@ describe('getBlockBotConversation', () => { theme: {}, messages: [], apiConfig: {}, + title: 'conversation_id', }; const result = getBlockBotConversation(conversation, isAssistantEnabled); expect(result.messages).toEqual(enterpriseMessaging); @@ -33,7 +34,6 @@ describe('getBlockBotConversation', () => { it('When conversation history and the last message is not enterprise messaging, appends enterprise messaging to conversation', () => { const conversation = { id: 'conversation_id', - theme: {}, messages: [ { role: 'user' as const, @@ -46,6 +46,7 @@ describe('getBlockBotConversation', () => { }, ], apiConfig: {}, + title: 'conversation_id', }; const result = getBlockBotConversation(conversation, isAssistantEnabled); expect(result.messages.length).toEqual(2); @@ -54,7 +55,7 @@ describe('getBlockBotConversation', () => { it('returns the conversation without changes when the last message is enterprise messaging', () => { const conversation = { id: 'conversation_id', - theme: {}, + title: 'conversation_id', messages: enterpriseMessaging, apiConfig: {}, }; @@ -66,7 +67,7 @@ describe('getBlockBotConversation', () => { it('returns the conversation with new enterprise message when conversation has enterprise messaging, but not as the last message', () => { const conversation = { id: 'conversation_id', - theme: {}, + title: 'conversation_id', messages: [ ...enterpriseMessaging, { @@ -91,7 +92,7 @@ describe('getBlockBotConversation', () => { it('when no conversation history, returns the welcome conversation', () => { const conversation = { id: 'conversation_id', - theme: {}, + title: 'conversation_id', messages: [], apiConfig: {}, }; @@ -101,7 +102,7 @@ describe('getBlockBotConversation', () => { it('returns a conversation history with the welcome conversation appended', () => { const conversation = { id: 'conversation_id', - theme: {}, + title: 'conversation_id', messages: [ { role: 'user' as const, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index 040082e3559e9..0700ccd40dbfd 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -22,6 +22,8 @@ import { useLocalStorage } from 'react-use'; import { PromptEditor } from './prompt_editor'; import { QuickPrompts } from './quick_prompts/quick_prompts'; import { mockAssistantAvailability, TestProviders } from '../mock/test_providers/test_providers'; +import { useFetchCurrentUserConversations } from './api'; +import { Conversation } from '../assistant_context/types'; jest.mock('../connectorland/use_load_connectors'); jest.mock('../connectorland/connector_setup'); @@ -29,24 +31,7 @@ jest.mock('react-use'); jest.mock('./prompt_editor', () => ({ PromptEditor: jest.fn() })); jest.mock('./quick_prompts/quick_prompts', () => ({ QuickPrompts: jest.fn() })); -jest.mock('./api/conversations/use_fetch_current_user_conversations', () => ({ - useFetchCurrentUserConversations: jest.fn().mockResolvedValue(() => ({ - [WELCOME_CONVERSATION_TITLE]: { - id: WELCOME_CONVERSATION_TITLE, - title: WELCOME_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - }, - [MOCK_CONVERSATION_TITLE]: { - id: MOCK_CONVERSATION_TITLE, - title: MOCK_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - }, - })), -})); - -const MOCK_CONVERSATION_TITLE = 'electric sheep'; +jest.mock('./api/conversations/use_fetch_current_user_conversations'); const renderAssistant = (extraProps = {}, providerProps = {}) => render( @@ -88,6 +73,25 @@ describe('Assistant', () => { data: connectors, } as unknown as UseQueryResult); + jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ + data: { + Welcome: { + id: 'Welcome', + title: 'Welcome', + messages: [], + apiConfig: {}, + }, + 'electric sheep': { + id: 'electric sheep', + title: 'electric sheep', + messages: [], + apiConfig: {}, + }, + }, + isLoading: false, + refetch: jest.fn(), + } as unknown as UseQueryResult, unknown>); + renderAssistant(); expect(persistToLocalStorage).toHaveBeenCalled(); @@ -112,24 +116,27 @@ describe('Assistant', () => { data: connectors, } as unknown as UseQueryResult); - const { getByLabelText } = renderAssistant( - {}, - { - getInitialConversations: () => ({ - [WELCOME_CONVERSATION_TITLE]: { - id: WELCOME_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - }, - [MOCK_CONVERSATION_TITLE]: { - id: MOCK_CONVERSATION_TITLE, - messages: [], - apiConfig: {}, - excludeFromLastConversationStorage: true, - }, - }), - } - ); + jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ + data: { + Welcome: { + id: 'Welcome', + title: 'Welcome', + messages: [], + apiConfig: {}, + }, + 'electric sheep': { + id: 'electric sheep', + title: 'electric sheep', + messages: [], + apiConfig: {}, + excludeFromLastConversationStorage: true, + }, + }, + isLoading: false, + refetch: jest.fn(), + } as unknown as UseQueryResult, unknown>); + + const { getByLabelText } = renderAssistant(); expect(persistToLocalStorage).toHaveBeenCalled(); @@ -152,6 +159,25 @@ describe('Assistant', () => { data: connectors, } as unknown as UseQueryResult); + jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ + data: { + Welcome: { + id: 'Welcome', + title: 'Welcome', + messages: [], + apiConfig: {}, + }, + 'electric sheep': { + id: 'electric sheep', + title: 'electric sheep', + messages: [], + apiConfig: {}, + }, + }, + isLoading: false, + refetch: jest.fn(), + } as unknown as UseQueryResult, unknown>); + renderAssistant({ setConversationId }); await act(async () => { @@ -171,6 +197,25 @@ describe('Assistant', () => { data: emptyConnectors, } as unknown as UseQueryResult); + jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ + data: { + Welcome: { + id: 'Welcome', + title: 'Welcome', + messages: [], + apiConfig: {}, + }, + 'electric sheep': { + id: 'electric sheep', + title: 'electric sheep', + messages: [], + apiConfig: {}, + }, + }, + isLoading: false, + refetch: jest.fn(), + } as unknown as UseQueryResult, unknown>); + renderAssistant(); expect(persistToLocalStorage).toHaveBeenCalled(); @@ -180,6 +225,25 @@ describe('Assistant', () => { describe('when not authorized', () => { it('should be disabled', async () => { + jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ + data: { + Welcome: { + id: 'Welcome', + title: 'Welcome', + messages: [], + apiConfig: {}, + }, + 'electric sheep': { + id: 'electric sheep', + title: 'electric sheep', + messages: [], + apiConfig: {}, + }, + }, + isLoading: false, + refetch: jest.fn(), + } as unknown as UseQueryResult, unknown>); + const { queryByTestId } = renderAssistant( {}, { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.test.tsx index 7573ac45e6aa5..b7de8a3a1626a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.test.tsx @@ -7,7 +7,7 @@ import { alertConvo, customConvo, welcomeConvo } from '../../mock/conversation'; import { useAssistantContext } from '../../assistant_context'; -import { fireEvent, render } from '@testing-library/react'; +import { fireEvent, render, act } from '@testing-library/react'; import { AssistantSettings, ANONYMIZATION_TAB, @@ -41,8 +41,8 @@ const mockContext = { selectedSettingsTab: 'CONVERSATIONS_TAB', }; const onClose = jest.fn(); -const onSave = jest.fn(); -const setSelectedConversationId = jest.fn(); +const onSave = jest.fn().mockResolvedValue(() => {}); +const onConversationSelected = jest.fn(); const testProps = { defaultConnectorId: '123', @@ -50,7 +50,8 @@ const testProps = { selectedConversation: welcomeConvo, onClose, onSave, - setSelectedConversationId, + onConversationSelected, + conversations: {}, }; jest.mock('../../assistant_context'); @@ -79,20 +80,25 @@ describe('AssistantSettings', () => { (useAssistantContext as jest.Mock).mockImplementation(() => mockContext); }); - it('saves changes', () => { + it('saves changes', async () => { const { getByTestId } = render(); - fireEvent.click(getByTestId('save-button')); + + await act(async () => { + fireEvent.click(getByTestId('save-button')); + }); expect(onSave).toHaveBeenCalled(); expect(saveSettings).toHaveBeenCalled(); }); - it('saves changes and updates selected conversation when selected conversation has been deleted', () => { + it('saves changes and updates selected conversation when selected conversation has been deleted', async () => { const { getByTestId } = render( ); - fireEvent.click(getByTestId('save-button')); + await act(async () => { + fireEvent.click(getByTestId('save-button')); + }); expect(onSave).toHaveBeenCalled(); - expect(setSelectedConversationId).toHaveBeenCalled(); + expect(onConversationSelected).toHaveBeenCalled(); expect(saveSettings).toHaveBeenCalled(); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index a3ed3d66e56ea..7f7ed54f5301b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -153,7 +153,7 @@ export const AssistantSettings: React.FC = React.memo( onConversationSelected(newSelectedConversationId); } await saveSettings(); - onSave(); + await onSave(); }, [ conversationSettings, defaultSelectedConversation.id, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.test.tsx index 506f937dd0ab6..9c1edbfb896c8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.test.tsx @@ -14,14 +14,17 @@ import { welcomeConvo } from '../../mock/conversation'; import { CONVERSATIONS_TAB } from './assistant_settings'; const setIsSettingsModalVisible = jest.fn(); -const setSelectedConversationId = jest.fn(); +const onConversationSelected = jest.fn(); + const testProps = { defaultConnectorId: '123', defaultProvider: OpenAiProviderType.OpenAi, isSettingsModalVisible: false, selectedConversation: welcomeConvo, setIsSettingsModalVisible, - setSelectedConversationId, + onConversationSelected, + conversations: {}, + refetchConversationsState: jest.fn(), }; const setSelectedSettingsTab = jest.fn(); const mockUseAssistantContext = { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index dbe2e6979eeb1..f7384529f9641 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -11,7 +11,33 @@ import { TestProviders } from '../../mock/test_providers/test_providers'; import { alertConvo, welcomeConvo } from '../../mock/conversation'; import React from 'react'; import { ConversationRole } from '../../assistant_context/types'; -import { updateConversationApi } from '../api'; + +/* import { + updateConversationApi as _updateConversationApi, + getConversationById as _getConversationById, + appendConversationMessagesApi as _appendConversationMessagesApi, +} from '../api/conversations'; +import { useMutation as _useMutation } from '@tanstack/react-query'; + + +const updateConversationApiMock = _updateConversationApi as jest.Mock; +const getConversationByIdMock = _getConversationById as jest.Mock; +const useMutationMock = _useMutation as jest.Mock; +*/ +jest.mock('../api/conversations', () => { + const actual = jest.requireActual('../api/conversations'); + return { + ...actual, + updateConversationApi: jest.fn((...args) => actual.updateConversationApi(...args)), + appendConversationMessagesApi: jest.fn((...args) => + actual.appendConversationMessagesApi(...args) + ), + createConversationApi: jest.fn((...args) => actual.createConversationApi(...args)), + deleteConversationApi: jest.fn((...args) => actual.deleteConversationApi(...args)), + getConversationById: jest.fn((...args) => actual.getConversationById(...args)), + }; +}); + const message = { content: 'You are a robot', role: 'user' as ConversationRole, @@ -136,7 +162,6 @@ describe('useConversation', () => { it('appends replacements', async () => { await act(async () => { - const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { wrapper: ({ children }) => {children}, }); @@ -151,7 +176,7 @@ describe('useConversation', () => { }, }); - expect(updateConversationApi).toHaveBeenCalledWith({ + expect(updateConversationApiMock).toHaveBeenCalledWith({ [alertConvo.id]: alertConvo, [welcomeConvo.id]: { ...welcomeConvo, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx index 961723378de22..a5b6c79f89d51 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx @@ -65,6 +65,7 @@ describe('ConnectorSelectorInline', () => { isDisabled={false} selectedConnectorId={undefined} selectedConversation={undefined} + onConnectorSelected={jest.fn()} /> ); @@ -76,6 +77,7 @@ describe('ConnectorSelectorInline', () => { id: 'conversation_id', messages: [], apiConfig: {}, + title: 'conversation_id', }; const { getByText } = render( @@ -83,6 +85,7 @@ describe('ConnectorSelectorInline', () => { isDisabled={false} selectedConnectorId={'missing-connector-id'} selectedConversation={conversation} + onConnectorSelected={jest.fn()} /> ); @@ -93,6 +96,7 @@ describe('ConnectorSelectorInline', () => { id: 'conversation_id', messages: [], apiConfig: {}, + title: 'conversation_id', }; const { getByTestId, queryByTestId } = render( @@ -100,6 +104,7 @@ describe('ConnectorSelectorInline', () => { isDisabled={false} selectedConnectorId={'missing-connector-id'} selectedConversation={conversation} + onConnectorSelected={jest.fn()} /> ); @@ -113,6 +118,7 @@ describe('ConnectorSelectorInline', () => { id: 'conversation_id', messages: [], apiConfig: {}, + title: 'conversation_id', }; const { getByTestId, queryByTestId } = render( @@ -120,6 +126,7 @@ describe('ConnectorSelectorInline', () => { isDisabled={false} selectedConnectorId={'missing-connector-id'} selectedConversation={conversation} + onConnectorSelected={jest.fn()} /> ); @@ -135,6 +142,7 @@ describe('ConnectorSelectorInline', () => { provider: 'OpenAI', }, conversationId: 'conversation_id', + title: 'conversation_id', }); }); it('On connector change to add new connector, onchange event does nothing', () => { @@ -142,6 +150,7 @@ describe('ConnectorSelectorInline', () => { id: 'conversation_id', messages: [], apiConfig: {}, + title: 'conversation_id', }; const { getByTestId } = render( @@ -149,6 +158,7 @@ describe('ConnectorSelectorInline', () => { isDisabled={false} selectedConnectorId={'missing-connector-id'} selectedConversation={conversation} + onConnectorSelected={jest.fn()} /> ); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 23c6cf1ef7ad4..a70034cfb71be 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -12598,13 +12598,6 @@ "xpack.elasticAssistant.assistant.conversationSelector.nextConversationTitle": "次の会話", "xpack.elasticAssistant.assistant.conversationSelector.placeholderTitle": "選択するか、入力して新規作成...", "xpack.elasticAssistant.assistant.conversationSelector.previousConversationTitle": "前の会話", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.ariaLabel": "会話セレクター", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.CustomOptionTextTitle": "新しい会話を作成：", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.defaultConversationTitle": "会話", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.deleteConversationTitle": "会話を削除", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.nextConversationTitle": "次の会話", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.placeholderTitle": "選択するか、入力して新規作成...", - "xpack.elasticAssistant.assistant.conversationSelectorSettings.previousConversationTitle": "前の会話", "xpack.elasticAssistant.assistant.dataAnonymizationEditor.contextEditor.allActionsTooltip": "すべてのアクション", "xpack.elasticAssistant.assistant.dataAnonymizationEditor.contextEditor.allowAction": "許可", "xpack.elasticAssistant.assistant.dataAnonymizationEditor.contextEditor.allowByDefaultAction": "デフォルトで許可", From 1d76ddfa4591667c6be41565b980ed0518347557 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 23 Jan 2024 23:31:33 +0000 Subject: [PATCH 034/141] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_fields.json | 23 ++ .../current_mappings.json | 351 +++++++++++------- 2 files changed, 231 insertions(+), 143 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 3a51ffd206ab0..ad3996c2ac30f 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -972,5 +972,28 @@ ], "cloud-security-posture-settings": [ "rules" + ], + "elastic-ai-assistant-prompts": [ + "content", + "created_at", + "created_by", + "id", + "is_default", + "is_new_conversation_default", + "is_shared", + "name", + "prompt_type", + "updated_at", + "updated_by" + ], + "elastic-ai-assistant-anonimization-fields": [ + "created_at", + "created_by", + "default_allow", + "default_allow_replacement", + "field_id", + "id", + "updated_at", + "updated_by" ] } diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 1c4c15482934a..400bbb5bf5146 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1058,21 +1058,6 @@ } } }, - "links": { - "dynamic": false, - "properties": { - "title": { - "type": "text" - }, - "description": { - "type": "text" - }, - "links": { - "dynamic": false, - "properties": {} - } - } - }, "lens": { "properties": { "title": { @@ -1106,32 +1091,6 @@ } } }, - "map": { - "properties": { - "description": { - "type": "text" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "mapStateJSON": { - "type": "text" - }, - "layerListJSON": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "bounds": { - "dynamic": false, - "properties": {} - } - } - }, "cases-comments": { "dynamic": false, "properties": { @@ -1461,18 +1420,25 @@ "dynamic": false, "properties": {} }, - "infrastructure-monitoring-log-view": { + "metrics-data-source": { + "dynamic": false, + "properties": {} + }, + "links": { "dynamic": false, "properties": { - "name": { + "title": { "type": "text" + }, + "description": { + "type": "text" + }, + "links": { + "dynamic": false, + "properties": {} } } }, - "metrics-data-source": { - "dynamic": false, - "properties": {} - }, "canvas-element": { "dynamic": false, "properties": { @@ -1564,10 +1530,10 @@ "prerelease_integrations_enabled": { "type": "boolean" }, - "output_secret_storage_requirements_met": { + "secret_storage_requirements_met": { "type": "boolean" }, - "secret_storage_requirements_met": { + "output_secret_storage_requirements_met": { "type": "boolean" } } @@ -2356,58 +2322,29 @@ "dynamic": false, "properties": {} }, - "slo": { - "dynamic": false, + "map": { "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "text" - }, "description": { "type": "text" }, - "indicator": { - "properties": { - "type": { - "type": "keyword" - }, - "params": { - "type": "flattened" - } - } - }, - "budgetingMethod": { - "type": "keyword" + "title": { + "type": "text" }, - "enabled": { - "type": "boolean" + "version": { + "type": "integer" }, - "tags": { - "type": "keyword" + "mapStateJSON": { + "type": "text" }, - "version": { - "type": "long" - } - } - }, - "threshold-explorer-view": { - "dynamic": false, - "properties": {} - }, - "observability-onboarding-state": { - "properties": { - "type": { - "type": "keyword" + "layerListJSON": { + "type": "text" }, - "state": { - "type": "object", - "dynamic": false + "uiStateJSON": { + "type": "text" }, - "progress": { - "type": "object", - "dynamic": false + "bounds": { + "dynamic": false, + "properties": {} } } }, @@ -2526,6 +2463,66 @@ } } }, + "infrastructure-monitoring-log-view": { + "dynamic": false, + "properties": { + "name": { + "type": "text" + } + } + }, + "enterprise_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "app_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "workplace_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "slo": { + "dynamic": false, + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "description": { + "type": "text" + }, + "indicator": { + "properties": { + "type": { + "type": "keyword" + }, + "params": { + "type": "flattened" + } + } + }, + "budgetingMethod": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "tags": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "threshold-explorer-view": { + "dynamic": false, + "properties": {} + }, "uptime-dynamic-settings": { "dynamic": false, "properties": {} @@ -2670,6 +2667,21 @@ "dynamic": false, "properties": {} }, + "observability-onboarding-state": { + "properties": { + "type": { + "type": "keyword" + }, + "state": { + "type": "object", + "dynamic": false + }, + "progress": { + "type": "object", + "dynamic": false + } + } + }, "infrastructure-ui-source": { "dynamic": false, "properties": {} @@ -2714,17 +2726,114 @@ } } }, - "enterprise_search_telemetry": { + "apm-telemetry": { "dynamic": false, "properties": {} }, - "app_search_telemetry": { - "dynamic": false, - "properties": {} + "apm-server-schema": { + "properties": { + "schemaJson": { + "type": "text", + "index": false + } + } }, - "workplace_search_telemetry": { - "dynamic": false, - "properties": {} + "apm-service-group": { + "properties": { + "groupName": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "description": { + "type": "text" + }, + "color": { + "type": "text" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "serviceEnvironmentFilterEnabled": { + "type": "boolean" + }, + "serviceNameFilterEnabled": { + "type": "boolean" + } + } + }, + "elastic-ai-assistant-prompts": { + "properties": { + "id": { + "type": "keyword" + }, + "is_default": { + "type": "boolean" + }, + "is_shared": { + "type": "boolean" + }, + "is_new_conversation_default": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "prompt_type": { + "type": "keyword" + }, + "content": { + "type": "keyword" + }, + "updated_at": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + } + } + }, + "elastic-ai-assistant-anonimization-fields": { + "properties": { + "id": { + "type": "keyword" + }, + "field_id": { + "type": "keyword" + }, + "default_allow": { + "type": "boolean" + }, + "default_allow_replacement": { + "type": "boolean" + }, + "updated_at": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + } + } }, "siem-ui-timeline-note": { "properties": { @@ -3187,49 +3296,5 @@ "index": false } } - }, - "apm-telemetry": { - "dynamic": false, - "properties": {} - }, - "apm-server-schema": { - "properties": { - "schemaJson": { - "type": "text", - "index": false - } - } - }, - "apm-service-group": { - "properties": { - "groupName": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "description": { - "type": "text" - }, - "color": { - "type": "text" - } - } - }, - "apm-custom-dashboards": { - "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" - } - } } } From 388576ed3b91427401031e52cb8b22aac5da4e99 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 23 Jan 2024 23:56:33 +0000 Subject: [PATCH 035/141] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 717c034a61b9d..7845beab46e49 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -83,6 +83,8 @@ describe('checking migration metadata changes on all registered SO types', () => "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "c151324d5f85178169395eecb12bac6b96064654", "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", + "elastic-ai-assistant-anonimization-fields": "04707fc69680fc95656f2438cdda1d70cbedf6bf", + "elastic-ai-assistant-prompts": "713a9d7e8f26b32ebb5c4042193ae29ba4059dd7", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a", From 0f36144a9a1cdc384a9174f63ed6b70ba4a8b05e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Jan 2024 00:42:19 +0000 Subject: [PATCH 036/141] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/conversations_data_client/get_conversation.test.ts | 2 +- .../conversations_data_client/update_conversation.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index c8f10acf556ba..d5f422abd7ded 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -21,7 +21,7 @@ describe('getConversation', () => { const data = getSearchListMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation(esClient, LIST_INDEX, id ); + const conversation = await getConversation(esClient, LIST_INDEX, id); const expected = getListResponseMock(); expect(conversation).toEqual(expected); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 8d799441291b1..b84ecf4b90dd8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -59,6 +59,8 @@ describe('updateConversation', () => { const options = getupdateConversationOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 0 }); - await expect(updateConversation({ ...options, esClient })).rejects.toThrow('No list has been updated'); + await expect(updateConversation({ ...options, esClient })).rejects.toThrow( + 'No list has been updated' + ); }); }); From c6ab1e5d01a90e868b0e48bdfaceb55858d1e56e Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 23 Jan 2024 20:07:34 -0800 Subject: [PATCH 037/141] fixed replacements test --- .../assistant/chat_send/use_chat_send.tsx | 2 +- .../impl/assistant/prompt/helpers.ts | 28 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index ab14aa845853c..81da68bd4ff16 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -68,7 +68,7 @@ export const useChatSend = ({ // Handles sending latest user prompt to API const handleSendMessage = useCallback( async (promptText: string) => { - const onNewReplacements = async (newReplacements: Record) => + const onNewReplacements = (newReplacements: Record) => appendReplacements({ conversationId: currentConversation.id, replacements: newReplacements, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index fa0cd0221bf0a..3e41bcc4bb3fe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -58,20 +58,22 @@ export async function getCombinedMessage({ selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; }): Promise { - const promptContextsContent = Object.keys(selectedPromptContexts) - .sort() - .map(async (id) => { - const promptContext = await transformRawData({ - allow: selectedPromptContexts[id].allow, - allowReplacement: selectedPromptContexts[id].allowReplacement, - currentReplacements, - getAnonymizedValue, - onNewReplacements, - rawData: selectedPromptContexts[id].rawData, - }); + const promptContextsContent = await Promise.all( + Object.keys(selectedPromptContexts) + .sort() + .map(async (id) => { + const promptContext = await transformRawData({ + allow: selectedPromptContexts[id].allow, + allowReplacement: selectedPromptContexts[id].allowReplacement, + currentReplacements, + getAnonymizedValue, + onNewReplacements, + rawData: selectedPromptContexts[id].rawData, + }); - return `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; - }); + return `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; + }) + ); return { content: `${ From 77abfd196a062e8e432b67911aeecbc8327a185b Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 23 Jan 2024 21:47:57 -0800 Subject: [PATCH 038/141] - --- ..._fetch_current_user_conversations.test.tsx | 51 +++++++++++ .../use_fetch_current_user_conversations.ts | 34 +++++-- .../use_settings_updater.test.tsx | 8 +- .../assistant/use_conversation/index.test.tsx | 89 +++++++++---------- 4 files changed, 124 insertions(+), 58 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx new file mode 100644 index 0000000000000..6af11ce6cef9b --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; + +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; +import React from 'react'; +import { + UseFetchCurrentUserConversationsParams, + useFetchCurrentUserConversations, +} from './use_fetch_current_user_conversations'; + +const statusResponse = { assistantModelEvaluation: true, assistantStreamingEnabled: false }; + +const http = { + fetch: jest.fn().mockResolvedValue(statusResponse), +}; +const onFetch = jest.fn(); + +const defaultProps = { http, onFetch } as unknown as UseFetchCurrentUserConversationsParams; + +const createWrapper = () => { + const queryClient = new QueryClient(); + // eslint-disable-next-line react/display-name + return ({ children }: { children: ReactNode }) => ( + {children} + ); +}; + +describe('useFetchCurrentUserConversations', () => { + it(`should make http request to fetch conversations`, () => { + renderHook(() => useFetchCurrentUserConversations(defaultProps), { + wrapper: createWrapper(), + }); + + expect(defaultProps.http.fetch).toHaveBeenCalledWith( + '/api/elastic_assistant/conversations/current_user/_find', + { + method: 'GET', + version: '2023-10-31', + signal: undefined, + } + ); + expect(onFetch).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts index 6814562c06a86..7ad0257d25e4e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts @@ -5,8 +5,7 @@ * 2.0. */ -import { CoreStart } from '@kbn/core/public'; -import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { HttpSetup } from '@kbn/core/public'; import { useQuery } from '@tanstack/react-query'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, @@ -21,10 +20,27 @@ export interface FetchConversationsResponse { data: Conversation[]; } -export const useFetchCurrentUserConversations = ( - onFetch: (result: FetchConversationsResponse) => Record -) => { - const { http } = useKibana().services; +export interface UseFetchCurrentUserConversationsParams { + http: HttpSetup; + onFetch: (result: FetchConversationsResponse) => Record; + signal?: AbortSignal | undefined; +} + +/** + * API call for fetching assistant conversations for the current user + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {Function} [options.onFetch] - transformation function for conversations fetch result + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {useQuery} hook for getting the status of the conversations + */ +export const useFetchCurrentUserConversations = ({ + http, + onFetch, + signal, +}: UseFetchCurrentUserConversationsParams) => { const query = { page: 1, perPage: 100, @@ -36,17 +52,17 @@ export const useFetchCurrentUserConversations = ( query.perPage, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ]; - const querySt = useQuery([cachingKeys, query], async () => { + + return useQuery([cachingKeys, query], async () => { const res = await http.fetch( ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, { method: 'GET', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, query, + signal, } ); return onFetch(res); }); - - return querySt; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx index 358f8cf36cd2e..bdd04fee89ae5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx @@ -146,7 +146,7 @@ describe('useSettingsUpdater', () => { setUpdatedDefaultAllowReplacement(updatedValues.defaultAllowReplacement); setUpdatedKnowledgeBaseSettings(updatedValues.knowledgeBase); - result.current.saveSettings(); + await result.current.saveSettings(); expect(mockHttp.fetch).toHaveBeenCalledWith( '/api/elastic_assistant/conversations/_bulk_action', @@ -173,7 +173,7 @@ describe('useSettingsUpdater', () => { setUpdatedKnowledgeBaseSettings(updatedValues.knowledgeBase); - result.current.saveSettings(); + await result.current.saveSettings(); expect(reportAssistantSettingToggled).toHaveBeenCalledWith({ isEnabledKnowledgeBase: false, isEnabledRAGAlerts: false, @@ -190,7 +190,7 @@ describe('useSettingsUpdater', () => { ...updatedValues.knowledgeBase, isEnabledKnowledgeBase: true, }); - result.current.saveSettings(); + await result.current.saveSettings(); expect(reportAssistantSettingToggled).toHaveBeenCalledWith({ isEnabledRAGAlerts: false, }); @@ -203,7 +203,7 @@ describe('useSettingsUpdater', () => { const { setUpdatedKnowledgeBaseSettings } = result.current; setUpdatedKnowledgeBaseSettings(mockValues.knowledgeBase); - result.current.saveSettings(); + await result.current.saveSettings(); expect(reportAssistantSettingToggled).not.toHaveBeenCalledWith(); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index f7384529f9641..f3c3c29d06cb5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -8,22 +8,17 @@ import { useConversation } from '.'; import { act, renderHook } from '@testing-library/react-hooks'; import { TestProviders } from '../../mock/test_providers/test_providers'; -import { alertConvo, welcomeConvo } from '../../mock/conversation'; +import { welcomeConvo } from '../../mock/conversation'; import React from 'react'; import { ConversationRole } from '../../assistant_context/types'; - -/* import { - updateConversationApi as _updateConversationApi, - getConversationById as _getConversationById, - appendConversationMessagesApi as _appendConversationMessagesApi, +import { + appendConversationMessagesApi, + deleteConversationApi, + getConversationById, + updateConversationApi, } from '../api/conversations'; -import { useMutation as _useMutation } from '@tanstack/react-query'; - +import { httpServiceMock } from '@kbn/core/public/mocks'; -const updateConversationApiMock = _updateConversationApi as jest.Mock; -const getConversationByIdMock = _getConversationById as jest.Mock; -const useMutationMock = _useMutation as jest.Mock; -*/ jest.mock('../api/conversations', () => { const actual = jest.requireActual('../api/conversations'); return { @@ -38,6 +33,11 @@ jest.mock('../api/conversations', () => { }; }); +const updateConversationApiMock = updateConversationApi as jest.Mock; +const getConversationByIdMock = getConversationById as jest.Mock; +const deleteConversationApiMock = deleteConversationApi as jest.Mock; +const appendConversationMessagesApiMock = appendConversationMessagesApi as jest.Mock; + const message = { content: 'You are a robot', role: 'user' as ConversationRole, @@ -57,7 +57,11 @@ const mockConvo = { }; describe('useConversation', () => { + let httpMock: ReturnType; + beforeEach(() => { + httpMock = httpServiceMock.createSetupContract({ basePath: '/test' }); + jest.clearAllMocks(); }); it('should append a message to an existing conversation when called with valid conversationId and message', async () => { @@ -134,15 +138,17 @@ describe('useConversation', () => { }); await waitForNextUpdate(); - const deleteResult = await result.current.deleteConversation('new-convo'); + await result.current.deleteConversation('new-convo'); - expect(deleteResult).toEqual(mockConvo); + expect(deleteConversationApiMock).toHaveBeenCalledWith({ + http: httpMock, + id: 'new-convo', + }); }); }); it('should update the apiConfig for an existing conversation when called with a valid conversationId and apiConfig', async () => { await act(async () => { - const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { wrapper: ({ children }) => {children}, }); @@ -154,8 +160,10 @@ describe('useConversation', () => { title: welcomeConvo.title, }); - expect(setConversations).toHaveBeenCalledWith({ - [welcomeConvo.id]: { ...welcomeConvo, apiConfig: mockConvo.apiConfig }, + expect(updateConversationApiMock).toHaveBeenCalledWith({ + http: httpMock, + conversationId: welcomeConvo.id, + apiConfig: mockConvo.apiConfig, }); }); }); @@ -167,6 +175,8 @@ describe('useConversation', () => { }); await waitForNextUpdate(); + getConversationByIdMock.mockResolvedValue(mockConvo); + await result.current.appendReplacements({ conversationId: welcomeConvo.id, replacements: { @@ -177,14 +187,12 @@ describe('useConversation', () => { }); expect(updateConversationApiMock).toHaveBeenCalledWith({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: { - ...welcomeConvo, - replacements: { - '1.0.0.721': '127.0.0.1', - '1.0.0.01': '10.0.0.1', - 'tsoh-tset': 'test-host', - }, + http: httpMock, + conversationId: welcomeConvo.id, + replacements: { + '1.0.0.721': '127.0.0.1', + '1.0.0.01': '10.0.0.1', + 'tsoh-tset': 'test-host', }, }); }); @@ -192,26 +200,21 @@ describe('useConversation', () => { it('should remove the last message from a conversation when called with valid conversationId', async () => { await act(async () => { - const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); + getConversationByIdMock.mockResolvedValue(mockConvo); + const removeResult = await result.current.removeLastMessage('new-convo'); expect(removeResult).toEqual([message]); - expect(setConversations).toHaveBeenCalledWith({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [mockConvo.id]: { ...mockConvo, messages: [message] }, - }); }); }); it('amendMessage updates the last message of conversation[] for a given conversationId with provided content', async () => { await act(async () => { - const setConversations = jest.fn(); const { result, waitForNextUpdate } = renderHook(() => useConversation(), { wrapper: ({ children }) => {children}, }); @@ -222,19 +225,15 @@ describe('useConversation', () => { content: 'hello world', }); - expect(setConversations).toHaveBeenCalledWith({ - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [mockConvo.id]: { - ...mockConvo, - messages: [ - message, - { - ...anotherMessage, - content: 'hello world', - }, - ], - }, + expect(appendConversationMessagesApiMock).toHaveBeenCalledWith({ + http: httpMock, + conversationId: mockConvo.id, + messages: [ + { + ...anotherMessage, + content: 'hello world', + }, + ], }); }); }); From 59704debc411e4a6e3f4cc0c74a2d07e5d880104 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 23 Jan 2024 21:54:10 -0800 Subject: [PATCH 039/141] fixed type checks --- .../impl/assistant/assistant_title/index.test.tsx | 1 + .../impl/assistant/use_conversation/helpers.test.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.test.tsx index bdc9f71e42fc5..fa5f4347d5069 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.test.tsx @@ -14,6 +14,7 @@ const testProps = { title: 'Test Title', docLinks: { ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', DOC_LINK_VERSION: '7.15' }, selectedConversation: undefined, + onChange: jest.fn(), }; describe('AssistantTitle', () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts index 8da6203512183..28f03dd4aab6e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts @@ -96,6 +96,7 @@ describe('useConversation helpers', () => { }, id: '1', messages: [], + title: '1', }; test('should return the conversation system prompt if it exists', () => { @@ -109,6 +110,7 @@ describe('useConversation helpers', () => { apiConfig: {}, id: '1', messages: [], + title: '1', }; const result = getDefaultSystemPrompt({ allSystemPrompts, @@ -123,6 +125,7 @@ describe('useConversation helpers', () => { apiConfig: {}, id: '4', // this id does not exist within allSystemPrompts messages: [], + title: '4', }; const result = getDefaultSystemPrompt({ allSystemPrompts, @@ -137,6 +140,7 @@ describe('useConversation helpers', () => { apiConfig: {}, id: '1', messages: [], + title: '1', }; const result = getDefaultSystemPrompt({ allSystemPrompts: allSystemPromptsNoDefault, @@ -151,6 +155,7 @@ describe('useConversation helpers', () => { apiConfig: {}, id: '1', messages: [], + title: '1', }; const result = getDefaultSystemPrompt({ allSystemPrompts: [], @@ -165,6 +170,7 @@ describe('useConversation helpers', () => { apiConfig: {}, id: '4', // this id does not exist within allSystemPrompts messages: [], + title: '1', }; const result = getDefaultSystemPrompt({ allSystemPrompts: allSystemPromptsNoDefault, From 0978a064fb0538f9fcd40bb4ddc0d1fbe7ba08ef Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Jan 2024 06:01:47 +0000 Subject: [PATCH 040/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/packages/kbn-elastic-assistant/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/packages/kbn-elastic-assistant/tsconfig.json b/x-pack/packages/kbn-elastic-assistant/tsconfig.json index f168c27aa3618..eea97cfc917dc 100644 --- a/x-pack/packages/kbn-elastic-assistant/tsconfig.json +++ b/x-pack/packages/kbn-elastic-assistant/tsconfig.json @@ -30,6 +30,5 @@ "@kbn/ui-theme", "@kbn/core-doc-links-browser", "@kbn/core", - "@kbn/kibana-react-plugin", ] } From 3129865aa888d1f0289323c41cf53071fcb37873 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 24 Jan 2024 13:51:55 -0800 Subject: [PATCH 041/141] - --- packages/kbn-data-stream-adapter/README.md | 69 --- packages/kbn-data-stream-adapter/index.ts | 20 - .../kbn-data-stream-adapter/jest.config.js | 13 - packages/kbn-data-stream-adapter/kibana.jsonc | 5 - packages/kbn-data-stream-adapter/package.json | 7 - ...reate_or_update_component_template.test.ts | 287 ------------- .../create_or_update_component_template.ts | 112 ----- .../src/create_or_update_data_stream.test.ts | 172 -------- .../src/create_or_update_data_stream.ts | 239 ----------- .../create_or_update_index_template.test.ts | 167 -------- .../src/create_or_update_index_template.ts | 66 --- .../src/data_stream_adapter.ts | 160 ------- .../src/data_stream_spaces_adapter.ts | 100 ----- .../src/field_maps/ecs_field_map.ts | 88 ---- .../field_maps/mapping_from_field_map.test.ts | 393 ------------------ .../src/field_maps/mapping_from_field_map.ts | 54 --- .../src/field_maps/types.ts | 56 --- .../src/install_with_timeout.test.ts | 63 --- .../src/install_with_timeout.ts | 67 --- .../src/resource_installer_utils.test.ts | 170 -------- .../src/resource_installer_utils.ts | 106 ----- .../src/retry_transient_es_errors.test.ts | 78 ---- .../src/retry_transient_es_errors.ts | 54 --- .../kbn-data-stream-adapter/tsconfig.json | 24 -- 24 files changed, 2570 deletions(-) delete mode 100644 packages/kbn-data-stream-adapter/README.md delete mode 100644 packages/kbn-data-stream-adapter/index.ts delete mode 100644 packages/kbn-data-stream-adapter/jest.config.js delete mode 100644 packages/kbn-data-stream-adapter/kibana.jsonc delete mode 100644 packages/kbn-data-stream-adapter/package.json delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts delete mode 100644 packages/kbn-data-stream-adapter/src/data_stream_adapter.ts delete mode 100644 packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts delete mode 100644 packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts delete mode 100644 packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts delete mode 100644 packages/kbn-data-stream-adapter/src/field_maps/types.ts delete mode 100644 packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/install_with_timeout.ts delete mode 100644 packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/resource_installer_utils.ts delete mode 100644 packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts delete mode 100644 packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts delete mode 100644 packages/kbn-data-stream-adapter/tsconfig.json diff --git a/packages/kbn-data-stream-adapter/README.md b/packages/kbn-data-stream-adapter/README.md deleted file mode 100644 index 04a3d854aced7..0000000000000 --- a/packages/kbn-data-stream-adapter/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# @kbn/data-stream-adapter - -Utility library for Elasticsearch data stream management. - -## DataStreamAdapter - -Manage single data streams. Example: - -``` -// Setup -const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); - -dataStream.setComponentTemplate({ - name: 'awesome-component-template', - fieldMap: { - 'awesome.field1: { type: 'keyword', required: true }, - 'awesome.nested.field2: { type: 'number', required: false }, - // ... - }, -}); - -dataStream.setIndexTemplate({ - name: 'awesome-index-template', - componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], - template: { - lifecycle: { - data_retention: '5d', - }, - }, -}); - -// Start -await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing. -``` - - -## DataStreamSpacesAdapter - -Manage data streams per space. Example: - -``` -// Setup -const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); - -spacesDataStream.setComponentTemplate({ - name: 'awesome-component-template', - fieldMap: { - 'awesome.field1: { type: 'keyword', required: true }, - 'awesome.nested.field2: { type: 'number', required: false }, - // ... - }, -}); - -spacesDataStream.setIndexTemplate({ - name: 'awesome-index-template', - componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], - template: { - lifecycle: { - data_retention: '5d', - }, - }, -}); - -// Start -await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams. - -// Create a space data stream on the fly -await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist. -``` diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts deleted file mode 100644 index 808145be4f12e..0000000000000 --- a/packages/kbn-data-stream-adapter/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export { DataStreamAdapter } from './src/data_stream_adapter'; -export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter'; -export { retryTransientEsErrors } from './src/retry_transient_es_errors'; -export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map'; - -export type { - DataStreamAdapterParams, - SetComponentTemplateParams, - SetIndexTemplateParams, - InstallParams, -} from './src/data_stream_adapter'; -export * from './src/field_maps/types'; diff --git a/packages/kbn-data-stream-adapter/jest.config.js b/packages/kbn-data-stream-adapter/jest.config.js deleted file mode 100644 index 48b717249e353..0000000000000 --- a/packages/kbn-data-stream-adapter/jest.config.js +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -module.exports = { - preset: '@kbn/test', - rootDir: '../..', - roots: ['/packages/kbn-data-stream-adapter'], -}; diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc deleted file mode 100644 index 99cbb458a8517..0000000000000 --- a/packages/kbn-data-stream-adapter/kibana.jsonc +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "shared-common", - "id": "@kbn/data-stream-adapter", - "owner": "@elastic/security-threat-hunting-explore" -} diff --git a/packages/kbn-data-stream-adapter/package.json b/packages/kbn-data-stream-adapter/package.json deleted file mode 100644 index 80b16c25ac217..0000000000000 --- a/packages/kbn-data-stream-adapter/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "@kbn/data-stream-adapter", - "version": "1.0.0", - "description": "Utility library for Elasticsearch Data Stream management", - "license": "SSPL-1.0 OR Elastic License 2.0", - "private": true -} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts deleted file mode 100644 index 3bd93b6bbcb08..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import type { DiagnosticResult } from '@elastic/elasticsearch'; -import { errors as EsErrors } from '@elastic/elasticsearch'; -import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; - -const randomDelayMultiplier = 0.01; -const logger = loggingSystemMock.createLogger(); -const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - -const componentTemplate = { - name: 'test-mappings', - _meta: { - managed: true, - }, - template: { - settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': 1500, - }, - mappings: { - dynamic: false, - properties: { - foo: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, -}; - -describe('createOrUpdateComponentTemplate', () => { - beforeEach(() => { - jest.resetAllMocks(); - jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); - }); - - it(`should call esClient to put component template`, async () => { - await createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }); - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledWith(componentTemplate); - }); - - it(`should retry on transient ES errors`, async () => { - clusterClient.cluster.putComponentTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - await createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }); - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); - }); - - it(`should log and throw error if max retries exceeded`, async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValue( - new EsErrors.ConnectionError('foo') - ); - await expect(() => - createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing component template test-mappings - foo` - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - }); - - it(`should log and throw error if ES throws error`, async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('generic error')); - - await expect(() => - createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing component template test-mappings - generic error` - ); - }); - - it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError({ - body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', - } as DiagnosticResult) - ); - const existingIndexTemplate = { - name: 'test-template', - index_template: { - index_patterns: ['test*'], - composed_of: ['test-mappings'], - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty-default`, - }, - 'index.mapping.total_fields.limit': 1800, - }, - mappings: { - dynamic: false, - }, - }, - }, - }; - - clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ - index_templates: [existingIndexTemplate], - }); - - await createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }); - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ - name: existingIndexTemplate.name, - body: { - ...existingIndexTemplate.index_template, - template: { - ...existingIndexTemplate.index_template.template, - settings: { - ...existingIndexTemplate.index_template.template?.settings, - 'index.mapping.total_fields.limit': 2500, - }, - }, - }, - }); - }); - - it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError({ - body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', - } as DiagnosticResult) - ); - const existingIndexTemplate = { - name: 'test-template', - index_template: { - index_patterns: ['test*'], - composed_of: ['test-mappings'], - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty-default`, - }, - 'index.mapping.total_fields.limit': 1800, - }, - mappings: { - dynamic: false, - }, - }, - }, - }; - - clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ - index_templates: [ - existingIndexTemplate, - { - name: 'lyndon', - // @ts-expect-error - index_template: { - index_patterns: ['intel*'], - }, - }, - { - name: 'sample_ds', - // @ts-expect-error - index_template: { - index_patterns: ['sample_ds-*'], - data_stream: { - hidden: false, - allow_custom_routing: false, - }, - }, - }, - ], - }); - - await createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }); - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ - name: existingIndexTemplate.name, - body: { - ...existingIndexTemplate.index_template, - template: { - ...existingIndexTemplate.index_template.template, - settings: { - ...existingIndexTemplate.index_template.template?.settings, - 'index.mapping.total_fields.limit': 2500, - }, - }, - }, - }); - }); - - it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError({ - body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded', - } as DiagnosticResult) - ); - const existingIndexTemplate = { - name: 'test-template', - index_template: { - index_patterns: ['test*'], - composed_of: ['test-mappings'], - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty-default`, - }, - 'index.mapping.total_fields.limit': 1800, - }, - mappings: { - dynamic: false, - }, - }, - }, - }; - clusterClient.indices.getIndexTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValueOnce({ - index_templates: [existingIndexTemplate], - }); - clusterClient.indices.putIndexTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - await createOrUpdateComponentTemplate({ - logger, - esClient: clusterClient, - template: componentTemplate, - totalFieldsLimit: 2500, - }); - - expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(3); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts deleted file mode 100644 index 9e6a1f2f788dd..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { - ClusterPutComponentTemplateRequest, - IndicesGetIndexTemplateIndexTemplateItem, -} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; -import { asyncForEach } from '@kbn/std'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; - -interface CreateOrUpdateComponentTemplateOpts { - logger: Logger; - esClient: ElasticsearchClient; - template: ClusterPutComponentTemplateRequest; - totalFieldsLimit: number; -} - -const putIndexTemplateTotalFieldsLimitUsingComponentTemplate = async ( - esClient: ElasticsearchClient, - componentTemplateName: string, - totalFieldsLimit: number, - logger: Logger -) => { - // Get all index templates and filter down to just the ones referencing this component template - const { index_templates: indexTemplates } = await retryTransientEsErrors( - () => esClient.indices.getIndexTemplate(), - { logger } - ); - const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter( - (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) => - (indexTemplate.index_template?.composed_of ?? []).includes(componentTemplateName) - ); - - await asyncForEach( - indexTemplatesUsingComponentTemplate, - async (template: IndicesGetIndexTemplateIndexTemplateItem) => { - await retryTransientEsErrors( - () => - esClient.indices.putIndexTemplate({ - name: template.name, - body: { - ...template.index_template, - template: { - ...template.index_template.template, - settings: { - ...template.index_template.template?.settings, - 'index.mapping.total_fields.limit': totalFieldsLimit, - }, - }, - }, - }), - { logger } - ); - } - ); -}; - -const createOrUpdateComponentTemplateHelper = async ( - esClient: ElasticsearchClient, - template: ClusterPutComponentTemplateRequest, - totalFieldsLimit: number, - logger: Logger -) => { - try { - await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger }); - } catch (error) { - if (error.message.match(/Limit of total fields \[\d+\] has been exceeded/) != null) { - // This error message occurs when there is an index template using this component template - // that contains a field limit setting that using this component template exceeds - // Specifically, this can happen for the ECS component template when we add new fields - // to adhere to the ECS spec. Individual index templates specify field limits so if the - // number of new ECS fields pushes the composed mapping above the limit, this error will - // occur. We have to update the field limit inside the index template now otherwise we - // can never update the component template - await putIndexTemplateTotalFieldsLimitUsingComponentTemplate( - esClient, - template.name, - totalFieldsLimit, - logger - ); - - // Try to update the component template again - await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { - logger, - }); - } else { - throw error; - } - } -}; - -export const createOrUpdateComponentTemplate = async ({ - logger, - esClient, - template, - totalFieldsLimit, -}: CreateOrUpdateComponentTemplateOpts) => { - logger.info(`Installing component template ${template.name}`); - - try { - await createOrUpdateComponentTemplateHelper(esClient, template, totalFieldsLimit, logger); - } catch (err) { - logger.error(`Error installing component template ${template.name} - ${err.message}`); - throw err; - } -}; diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts deleted file mode 100644 index cc587dcaebfad..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { - updateDataStreams, - createDataStream, - createOrUpdateDataStream, -} from './create_or_update_data_stream'; - -const logger = loggingSystemMock.createLogger(); -const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - -esClient.indices.putMapping.mockResolvedValue({ acknowledged: true }); -esClient.indices.putSettings.mockResolvedValue({ acknowledged: true }); - -const simulateIndexTemplateResponse = { template: { mappings: { is_managed: true } } }; -esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse); - -const name = 'test_data_stream'; -const totalFieldsLimit = 1000; - -describe('updateDataStreams', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it(`should update data streams`, async () => { - const dataStreamName = 'test_data_stream-default'; - esClient.indices.getDataStream.mockResolvedValueOnce({ - data_streams: [{ name: dataStreamName } as IndicesDataStream], - }); - - await updateDataStreams({ - esClient, - logger, - name, - totalFieldsLimit, - }); - - expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); - - expect(esClient.indices.putSettings).toHaveBeenCalledWith({ - index: dataStreamName, - body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, - }); - expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ - name: dataStreamName, - }); - expect(esClient.indices.putMapping).toHaveBeenCalledWith({ - index: dataStreamName, - body: simulateIndexTemplateResponse.template.mappings, - }); - }); - - it(`should update multiple data streams`, async () => { - const dataStreamName1 = 'test_data_stream-1'; - const dataStreamName2 = 'test_data_stream-2'; - esClient.indices.getDataStream.mockResolvedValueOnce({ - data_streams: [{ name: dataStreamName1 }, { name: dataStreamName2 }] as IndicesDataStream[], - }); - - await updateDataStreams({ - esClient, - logger, - name, - totalFieldsLimit, - }); - - expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2); - expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); - expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2); - }); - - it(`should not update data streams when not exist`, async () => { - esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); - - await updateDataStreams({ - esClient, - logger, - name, - totalFieldsLimit, - }); - - expect(esClient.indices.putSettings).not.toHaveBeenCalled(); - expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(esClient.indices.putMapping).not.toHaveBeenCalled(); - }); -}); - -describe('createDataStream', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it(`should create data stream`, async () => { - esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); - - await createDataStream({ - esClient, - logger, - name, - }); - - expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); - }); - - it(`should not create data stream if already exists`, async () => { - esClient.indices.getDataStream.mockResolvedValueOnce({ - data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], - }); - - await createDataStream({ - esClient, - logger, - name, - }); - - expect(esClient.indices.createDataStream).not.toHaveBeenCalled(); - }); -}); - -describe('createOrUpdateDataStream', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it(`should create data stream if not exists`, async () => { - esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); - - await createDataStream({ - esClient, - logger, - name, - }); - - expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); - }); - - it(`should update data stream if already exists`, async () => { - esClient.indices.getDataStream.mockResolvedValueOnce({ - data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], - }); - - await createOrUpdateDataStream({ - esClient, - logger, - name, - totalFieldsLimit, - }); - - expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); - - expect(esClient.indices.putSettings).toHaveBeenCalledWith({ - index: name, - body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, - }); - expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ - name, - }); - expect(esClient.indices.putMapping).toHaveBeenCalledWith({ - index: name, - body: simulateIndexTemplateResponse.template.mappings, - }); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts deleted file mode 100644 index 5cff6005ea8e0..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; -import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; -import { get } from 'lodash'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; - -interface UpdateIndexMappingsOpts { - logger: Logger; - esClient: ElasticsearchClient; - indexNames: string[]; - totalFieldsLimit: number; -} - -interface UpdateIndexOpts { - logger: Logger; - esClient: ElasticsearchClient; - indexName: string; - totalFieldsLimit: number; -} - -const updateTotalFieldLimitSetting = async ({ - logger, - esClient, - indexName, - totalFieldsLimit, -}: UpdateIndexOpts) => { - logger.debug(`Updating total field limit setting for ${indexName} data stream.`); - - try { - const body = { 'index.mapping.total_fields.limit': totalFieldsLimit }; - await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), { - logger, - }); - } catch (err) { - logger.error( - `Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}` - ); - throw err; - } -}; - -// This will update the mappings but *not* the settings. This -// is due to the fact settings can be classed as dynamic and static, and static -// updates will fail on an index that isn't closed. New settings *will* be applied as part -// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 -const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => { - logger.debug(`Updating mappings for ${indexName} data stream.`); - - let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; - try { - simulatedIndexMapping = await retryTransientEsErrors( - () => esClient.indices.simulateIndexTemplate({ name: indexName }), - { logger } - ); - } catch (err) { - logger.error( - `Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}` - ); - return; - } - - const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); - - if (simulatedMapping == null) { - logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`); - return; - } - - try { - await retryTransientEsErrors( - () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }), - { logger } - ); - } catch (err) { - logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`); - throw err; - } -}; -/** - * Updates the data stream mapping and total field limit setting - */ -const updateDataStreamMappings = async ({ - logger, - esClient, - totalFieldsLimit, - indexNames, -}: UpdateIndexMappingsOpts) => { - // Update total field limit setting of found indices - // Other index setting changes are not updated at this time - await Promise.all( - indexNames.map((indexName) => - updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName }) - ) - ); - // Update mappings of the found indices. - await Promise.all( - indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName })) - ); -}; - -export interface CreateOrUpdateDataStreamParams { - name: string; - logger: Logger; - esClient: ElasticsearchClient; - totalFieldsLimit: number; -} - -export async function createOrUpdateDataStream({ - logger, - esClient, - name, - totalFieldsLimit, -}: CreateOrUpdateDataStreamParams): Promise { - logger.info(`Creating data stream - ${name}`); - - // check if data stream exists - let dataStreamExists = false; - try { - const response = await retryTransientEsErrors( - () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), - { logger } - ); - dataStreamExists = response.data_streams.length > 0; - } catch (error) { - if (error?.statusCode !== 404) { - logger.error(`Error fetching data stream for ${name} - ${error.message}`); - throw error; - } - } - - // if a data stream exists, update the underlying mapping - if (dataStreamExists) { - await updateDataStreamMappings({ - logger, - esClient, - indexNames: [name], - totalFieldsLimit, - }); - } else { - try { - await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); - } catch (error) { - if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { - logger.error(`Error creating data stream ${name} - ${error.message}`); - throw error; - } - } - } -} - -export interface CreateDataStreamParams { - name: string; - logger: Logger; - esClient: ElasticsearchClient; -} - -export async function createDataStream({ - logger, - esClient, - name, -}: CreateDataStreamParams): Promise { - logger.info(`Creating data stream - ${name}`); - - // check if data stream exists - let dataStreamExists = false; - try { - const response = await retryTransientEsErrors( - () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), - { logger } - ); - dataStreamExists = response.data_streams.length > 0; - } catch (error) { - if (error?.statusCode !== 404) { - logger.error(`Error fetching data stream for ${name} - ${error.message}`); - throw error; - } - } - - // return if data stream already created - if (dataStreamExists) { - return; - } - - try { - await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); - } catch (error) { - if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { - logger.error(`Error creating data stream ${name} - ${error.message}`); - throw error; - } - } -} - -export interface CreateOrUpdateSpacesDataStreamParams { - name: string; - logger: Logger; - esClient: ElasticsearchClient; - totalFieldsLimit: number; -} - -export async function updateDataStreams({ - logger, - esClient, - name, - totalFieldsLimit, -}: CreateOrUpdateSpacesDataStreamParams): Promise { - logger.info(`Updating data streams - ${name}`); - - // check if data stream exists - let dataStreams: IndicesDataStream[] = []; - try { - const response = await retryTransientEsErrors( - () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), - { logger } - ); - dataStreams = response.data_streams; - } catch (error) { - if (error?.statusCode !== 404) { - logger.error(`Error fetching data stream for ${name} - ${error.message}`); - throw error; - } - } - if (dataStreams.length > 0) { - await updateDataStreamMappings({ - logger, - esClient, - totalFieldsLimit, - indexNames: dataStreams.map((dataStream) => dataStream.name), - }); - } -} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts deleted file mode 100644 index cb3b6e77a02b5..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { errors as EsErrors } from '@elastic/elasticsearch'; -import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; - -const randomDelayMultiplier = 0.01; -const logger = loggingSystemMock.createLogger(); -const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - -const getIndexTemplate = (namespace: string = 'default', useDataStream: boolean = false) => ({ - name: `.alerts-test.alerts-${namespace}-index-template`, - body: { - _meta: { - kibana: { - version: '8.6.1', - }, - managed: true, - namespace, - }, - composed_of: ['mappings1', 'framework-mappings'], - index_patterns: [`.internal.alerts-test.alerts-${namespace}-*`], - template: { - mappings: { - _meta: { - kibana: { - version: '8.6.1', - }, - managed: true, - namespace, - }, - dynamic: false, - }, - settings: { - auto_expand_replicas: '0-1', - hidden: true, - ...(useDataStream - ? {} - : { - 'index.lifecycle': { - name: 'test-ilm-policy', - rollover_alias: `.alerts-test.alerts-${namespace}`, - }, - }), - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': 2500, - }, - }, - priority: namespace.length, - }, -}); - -const simulateTemplateResponse = { - template: { - aliases: { - alias_name_1: { - is_hidden: true, - }, - alias_name_2: { - is_hidden: true, - }, - }, - mappings: { enabled: false }, - settings: {}, - }, -}; - -describe('createOrUpdateIndexTemplate', () => { - beforeEach(() => { - jest.resetAllMocks(); - jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); - }); - - it(`should call esClient to put index template`, async () => { - esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); - await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); - - expect(esClient.indices.simulateTemplate).toHaveBeenCalledWith(getIndexTemplate()); - expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith(getIndexTemplate()); - }); - - it(`should retry on transient ES errors`, async () => { - esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); - esClient.indices.putIndexTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); - - expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); - }); - - it(`should retry simulateTemplate on transient ES errors`, async () => { - esClient.indices.simulateTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockImplementation(async () => simulateTemplateResponse); - esClient.indices.putIndexTemplate.mockResolvedValue({ acknowledged: true }); - await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate }); - - expect(esClient.indices.simulateTemplate).toHaveBeenCalledTimes(3); - }); - - it(`should log and throw error if max retries exceeded`, async () => { - esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); - esClient.indices.putIndexTemplate.mockRejectedValue(new EsErrors.ConnectionError('foo')); - await expect(() => - createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing index template .alerts-test.alerts-default-index-template - foo`, - expect.any(Error) - ); - expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(4); - }); - - it(`should log and throw error if ES throws error`, async () => { - esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); - esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); - - await expect(() => - createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing index template .alerts-test.alerts-default-index-template - generic error`, - expect.any(Error) - ); - }); - - it(`should log and return without updating template if simulate throws error`, async () => { - esClient.indices.simulateTemplate.mockRejectedValue(new Error('simulate error')); - esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); - - await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); - - expect(logger.error).toHaveBeenCalledWith( - `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - simulate error`, - expect.any(Error) - ); - expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - }); - - it(`should throw error if simulate returns empty mappings`, async () => { - esClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ - ...simulateTemplateResponse, - template: { - ...simulateTemplateResponse.template, - mappings: {}, - }, - })); - - await expect(() => - createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping"` - ); - expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts deleted file mode 100644 index c80abf3c8045e..0000000000000 --- a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { - IndicesPutIndexTemplateRequest, - MappingTypeMapping, -} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; -import { isEmpty } from 'lodash'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; - -interface CreateOrUpdateIndexTemplateOpts { - logger: Logger; - esClient: ElasticsearchClient; - template: IndicesPutIndexTemplateRequest; -} - -/** - * Installs index template that uses installed component template - * Prior to installation, simulates the installation to check for possible - * conflicts. Simulate should return an empty mapping if a template - * conflicts with an already installed template. - */ -export const createOrUpdateIndexTemplate = async ({ - logger, - esClient, - template, -}: CreateOrUpdateIndexTemplateOpts) => { - logger.info(`Installing index template ${template.name}`); - - let mappings: MappingTypeMapping = {}; - try { - // Simulate the index template to proactively identify any issues with the mapping - const simulateResponse = await retryTransientEsErrors( - () => esClient.indices.simulateTemplate(template), - { logger } - ); - mappings = simulateResponse.template.mappings; - } catch (err) { - logger.error( - `Failed to simulate index template mappings for ${template.name}; not applying mappings - ${err.message}`, - err - ); - return; - } - - if (isEmpty(mappings)) { - throw new Error( - `No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping` - ); - } - - try { - await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { - logger, - }); - } catch (err) { - logger.error(`Error installing index template ${template.name} - ${err.message}`, err); - throw err; - } -}; diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts deleted file mode 100644 index 3b3e2958eb46a..0000000000000 --- a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { - ClusterPutComponentTemplateRequest, - IndicesIndexSettings, - IndicesPutIndexTemplateIndexTemplateMapping, - IndicesPutIndexTemplateRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; -import type { Subject } from 'rxjs'; -import type { FieldMap } from './field_maps/types'; -import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; -import { createOrUpdateDataStream } from './create_or_update_data_stream'; -import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; -import { InstallShutdownError, installWithTimeout } from './install_with_timeout'; -import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils'; - -export interface DataStreamAdapterParams { - kibanaVersion: string; - totalFieldsLimit?: number; -} -export interface SetComponentTemplateParams { - name: string; - fieldMap: FieldMap; - settings?: IndicesIndexSettings; - dynamic?: 'strict' | boolean; -} -export interface SetIndexTemplateParams { - name: string; - componentTemplateRefs?: string[]; - namespace?: string; - template?: IndicesPutIndexTemplateIndexTemplateMapping; - hidden?: boolean; -} - -export interface GetInstallFnParams { - logger: Logger; - pluginStop$: Subject; - tasksTimeoutMs?: number; -} -export interface InstallParams { - logger: Logger; - esClient: ElasticsearchClient | Promise; - pluginStop$: Subject; - tasksTimeoutMs?: number; -} - -const DEFAULT_FIELDS_LIMIT = 2500; - -export class DataStreamAdapter { - protected readonly kibanaVersion: string; - protected readonly totalFieldsLimit: number; - protected componentTemplates: ClusterPutComponentTemplateRequest[] = []; - protected indexTemplates: IndicesPutIndexTemplateRequest[] = []; - protected installed: boolean; - - constructor(protected readonly name: string, options: DataStreamAdapterParams) { - this.installed = false; - this.kibanaVersion = options.kibanaVersion; - this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT; - } - - public setComponentTemplate(params: SetComponentTemplateParams) { - if (this.installed) { - throw new Error('Cannot set component template after install'); - } - this.componentTemplates.push(getComponentTemplate(params)); - } - - public setIndexTemplate(params: SetIndexTemplateParams) { - if (this.installed) { - throw new Error('Cannot set index template after install'); - } - this.indexTemplates.push( - getIndexTemplate({ - ...params, - indexPatterns: [this.name], - kibanaVersion: this.kibanaVersion, - totalFieldsLimit: this.totalFieldsLimit, - }) - ); - } - - protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) { - return async (promise: Promise, description?: string): Promise => { - try { - await installWithTimeout({ - installFn: () => promise, - description, - timeoutMs: tasksTimeoutMs, - pluginStop$, - }); - } catch (err) { - if (err instanceof InstallShutdownError) { - logger.info(err.message); - } else { - throw err; - } - } - }; - } - - public async install({ - logger, - esClient: esClientToResolve, - pluginStop$, - tasksTimeoutMs, - }: InstallParams) { - this.installed = true; - - const esClient = await esClientToResolve; - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); - - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `${componentTemplate.name} component template` - ) - ) - ); - - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ - template: indexTemplate, - esClient, - logger, - }), - `${indexTemplate.name} index template` - ) - ) - ); - - // create data stream when everything is ready - await installFn( - createOrUpdateDataStream({ - name: this.name, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `${this.name} data stream` - ); - } -} diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts deleted file mode 100644 index 5daad080d4720..0000000000000 --- a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; -import { createDataStream, updateDataStreams } from './create_or_update_data_stream'; -import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; -import { - DataStreamAdapter, - type DataStreamAdapterParams, - type InstallParams, -} from './data_stream_adapter'; - -export class DataStreamSpacesAdapter extends DataStreamAdapter { - private installedSpaceDataStreamName: Map>; - private _installSpace?: (spaceId: string) => Promise; - - constructor(private readonly prefix: string, options: DataStreamAdapterParams) { - super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names - this.installedSpaceDataStreamName = new Map(); - } - - public async install({ - logger, - esClient: esClientToResolve, - pluginStop$, - tasksTimeoutMs, - }: InstallParams) { - this.installed = true; - - const esClient = await esClientToResolve; - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); - - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `create or update ${componentTemplate.name} component template` - ) - ) - ); - - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), - `create or update ${indexTemplate.name} index template` - ) - ) - ); - - // Update existing space data streams - await installFn( - updateDataStreams({ - name: `${this.prefix}-*`, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `update space data streams` - ); - - // define function to install data stream for spaces on demand - this._installSpace = async (spaceId: string) => { - const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); - if (existingInstallPromise) { - return existingInstallPromise; - } - const name = `${this.prefix}-${spaceId}`; - const installPromise = installFn( - createDataStream({ name, esClient, logger }), - `create ${name} data stream` - ).then(() => name); - - this.installedSpaceDataStreamName.set(spaceId, installPromise); - return installPromise; - }; - } - - public async installSpace(spaceId: string): Promise { - if (!this._installSpace) { - throw new Error('Cannot installSpace before install'); - } - return this._installSpace(spaceId); - } - - public async getInstalledSpaceName(spaceId: string): Promise { - return this.installedSpaceDataStreamName.get(spaceId); - } -} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts deleted file mode 100644 index 17e8af1da7887..0000000000000 --- a/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { EcsFlat } from '@kbn/ecs'; -import type { EcsMetadata, FieldMap } from './types'; - -const EXCLUDED_TYPES = ['constant_keyword']; - -// ECS fields that have reached Stage 2 in the RFC process -// are included in the generated Yaml but are still considered -// experimental. Some are correctly marked as beta but most are -// not. - -// More about the RFC stages here: https://elastic.github.io/ecs/stages.html - -// The following RFCS are currently in stage 2: -// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md - -// Fields from these RFCs that are not already in the ECS component template -// as of 8.11 are manually identified as experimental below. -// The next time this list is updated, we should check the above list of RFCs to -// see if any have moved to Stage 3 and remove them from the list and check if -// there are any new stage 2 RFCs with fields we should exclude as experimental. - -const EXPERIMENTAL_FIELDS = [ - 'faas.trigger', // this was previously mapped as nested but changed to object - 'faas.trigger.request_id', - 'faas.trigger.type', - 'host.cpu.system.norm.pct', - 'host.cpu.user.norm.pct', - 'host.fsstats.total_size.total', - 'host.fsstats.total_size.used', - 'host.fsstats.total_size.used.pct', - 'host.load.norm.1', - 'host.load.norm.5', - 'host.load.norm.15', - 'host.memory.actual.used.bytes', - 'host.memory.actual.used.pct', - 'host.memory.total', - 'process.io.bytes', - 'volume.bus_type', - 'volume.default_access', - 'volume.device_name', - 'volume.device_type', - 'volume.dos_name', - 'volume.file_system_type', - 'volume.mount_name', - 'volume.nt_name', - 'volume.product_id', - 'volume.product_name', - 'volume.removable', - 'volume.serial_number', - 'volume.size', - 'volume.vendor_id', - 'volume.vendor_name', - 'volume.writable', -]; - -export const ecsFieldMap: FieldMap = Object.fromEntries( - Object.entries(EcsFlat) - .filter( - ([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key) - ) - .map(([key, _]) => { - const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat]; - return [ - key, - { - type: value.type, - array: value.normalize.includes('array'), - required: !!value.required, - ...(value.scaling_factor ? { scaling_factor: value.scaling_factor } : {}), - ...(value.ignore_above ? { ignore_above: value.ignore_above } : {}), - ...(value.multi_fields ? { multi_fields: value.multi_fields } : {}), - }, - ]; - }) -); - -export type EcsFieldMap = typeof ecsFieldMap; diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts deleted file mode 100644 index e851bdc21d01b..0000000000000 --- a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; -import { mappingFromFieldMap } from './mapping_from_field_map'; - -export const testFieldMap: FieldMap = { - date_field: { - type: 'date', - array: false, - required: true, - }, - keyword_field: { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - long_field: { - type: 'long', - array: false, - required: false, - }, - multifield_field: { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - multi_fields: [ - { - flat_name: 'multifield_field.text', - name: 'text', - type: 'match_only_text', - }, - ], - }, - geopoint_field: { - type: 'geo_point', - array: false, - required: false, - }, - ip_field: { - type: 'ip', - array: false, - required: false, - }, - array_field: { - type: 'keyword', - array: true, - required: false, - ignore_above: 1024, - }, - nested_array_field: { - type: 'nested', - array: false, - required: false, - }, - 'nested_array_field.field1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'nested_array_field.field2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - scaled_float_field: { - type: 'scaled_float', - array: false, - required: false, - scaling_factor: 1000, - }, - constant_keyword_field: { - type: 'constant_keyword', - array: false, - required: false, - }, - 'parent_field.child1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'parent_field.child2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - unmapped_object: { - type: 'object', - required: false, - enabled: false, - }, - formatted_field: { - type: 'date_range', - required: false, - format: 'epoch_millis||strict_date_optional_time', - }, -}; -export const expectedTestMapping = { - properties: { - array_field: { - ignore_above: 1024, - type: 'keyword', - }, - constant_keyword_field: { - type: 'constant_keyword', - }, - date_field: { - type: 'date', - }, - multifield_field: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - geopoint_field: { - type: 'geo_point', - }, - ip_field: { - type: 'ip', - }, - keyword_field: { - ignore_above: 1024, - type: 'keyword', - }, - long_field: { - type: 'long', - }, - nested_array_field: { - properties: { - field1: { - ignore_above: 1024, - type: 'keyword', - }, - field2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - type: 'nested', - }, - parent_field: { - properties: { - child1: { - ignore_above: 1024, - type: 'keyword', - }, - child2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - scaled_float_field: { - scaling_factor: 1000, - type: 'scaled_float', - }, - unmapped_object: { - enabled: false, - type: 'object', - }, - formatted_field: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - }, -}; - -describe('mappingFromFieldMap', () => { - it('correctly creates mapping from field map', () => { - expect(mappingFromFieldMap(testFieldMap)).toEqual({ - dynamic: 'strict', - ...expectedTestMapping, - }); - expect(mappingFromFieldMap(alertFieldMap)).toEqual({ - dynamic: 'strict', - properties: { - '@timestamp': { - ignore_malformed: false, - type: 'date', - }, - event: { - properties: { - action: { - type: 'keyword', - }, - kind: { - type: 'keyword', - }, - }, - }, - kibana: { - properties: { - alert: { - properties: { - action_group: { - type: 'keyword', - }, - case_ids: { - type: 'keyword', - }, - duration: { - properties: { - us: { - type: 'long', - }, - }, - }, - end: { - type: 'date', - }, - flapping: { - type: 'boolean', - }, - flapping_history: { - type: 'boolean', - }, - maintenance_window_ids: { - type: 'keyword', - }, - instance: { - properties: { - id: { - type: 'keyword', - }, - }, - }, - last_detected: { - type: 'date', - }, - reason: { - fields: { - text: { - type: 'match_only_text', - }, - }, - type: 'keyword', - }, - rule: { - properties: { - category: { - type: 'keyword', - }, - consumer: { - type: 'keyword', - }, - execution: { - properties: { - uuid: { - type: 'keyword', - }, - }, - }, - name: { - type: 'keyword', - }, - parameters: { - type: 'flattened', - ignore_above: 4096, - }, - producer: { - type: 'keyword', - }, - revision: { - type: 'long', - }, - rule_type_id: { - type: 'keyword', - }, - tags: { - type: 'keyword', - }, - uuid: { - type: 'keyword', - }, - }, - }, - start: { - type: 'date', - }, - status: { - type: 'keyword', - }, - time_range: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - url: { - ignore_above: 2048, - index: false, - type: 'keyword', - }, - uuid: { - type: 'keyword', - }, - workflow_assignee_ids: { - type: 'keyword', - }, - workflow_status: { - type: 'keyword', - }, - workflow_tags: { - type: 'keyword', - }, - }, - }, - space_ids: { - type: 'keyword', - }, - version: { - type: 'version', - }, - }, - }, - tags: { - type: 'keyword', - }, - }, - }); - expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ - dynamic: 'strict', - properties: { - kibana: { - properties: { - alert: { - properties: { - risk_score: { type: 'float' }, - rule: { - properties: { - author: { type: 'keyword' }, - created_at: { type: 'date' }, - created_by: { type: 'keyword' }, - description: { type: 'keyword' }, - enabled: { type: 'keyword' }, - from: { type: 'keyword' }, - interval: { type: 'keyword' }, - license: { type: 'keyword' }, - note: { type: 'keyword' }, - references: { type: 'keyword' }, - rule_id: { type: 'keyword' }, - rule_name_override: { type: 'keyword' }, - to: { type: 'keyword' }, - type: { type: 'keyword' }, - updated_at: { type: 'date' }, - updated_by: { type: 'keyword' }, - version: { type: 'keyword' }, - }, - }, - severity: { type: 'keyword' }, - suppression: { - properties: { - docs_count: { type: 'long' }, - end: { type: 'date' }, - terms: { - properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, - }, - start: { type: 'date' }, - }, - }, - system_status: { type: 'keyword' }, - workflow_reason: { type: 'keyword' }, - workflow_status_updated_at: { type: 'date' }, - workflow_user: { type: 'keyword' }, - }, - }, - }, - }, - ecs: { properties: { version: { type: 'keyword' } } }, - }, - }); - }); - - it('uses dynamic setting if specified', () => { - expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ - dynamic: true, - ...expectedTestMapping, - }); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts deleted file mode 100644 index 5878cedd44195..0000000000000 --- a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { set } from '@kbn/safer-lodash-set'; -import type { FieldMap, MultiField } from './types'; - -export function mappingFromFieldMap( - fieldMap: FieldMap, - dynamic: 'strict' | boolean = 'strict' -): MappingTypeMapping { - const mappings = { - dynamic, - properties: {}, - }; - - const fields = Object.keys(fieldMap).map((key: string) => { - const field = fieldMap[key]; - return { - name: key, - ...field, - }; - }); - - fields.forEach((field) => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, required, array, multi_fields, ...rest } = field; - const mapped = multi_fields - ? { - ...rest, - // eslint-disable-next-line @typescript-eslint/naming-convention - fields: multi_fields.reduce((acc, multi_field: MultiField) => { - acc[multi_field.name] = { - type: multi_field.type, - }; - return acc; - }, {} as Record), - } - : rest; - - set(mappings.properties, field.name.split('.').join('.properties.'), mapped); - - if (name === '@timestamp') { - set(mappings.properties, `${name}.ignore_malformed`, false); - } - }); - - return mappings; -} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/types.ts b/packages/kbn-data-stream-adapter/src/field_maps/types.ts deleted file mode 100644 index 0a0b68a2f26e6..0000000000000 --- a/packages/kbn-data-stream-adapter/src/field_maps/types.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export interface AllowedValue { - description?: string; - name?: string; -} - -export interface MultiField { - flat_name: string; - name: string; - type: string; -} - -export interface EcsMetadata { - allowed_values?: AllowedValue[]; - dashed_name: string; - description: string; - doc_values?: boolean; - example?: string | number | boolean; - flat_name: string; - ignore_above?: number; - index?: boolean; - level: string; - multi_fields?: MultiField[]; - name: string; - normalize: string[]; - required?: boolean; - scaling_factor?: number; - short: string; - type: string; - properties?: Record; -} - -export interface FieldMap { - [key: string]: { - type: string; - required: boolean; - array?: boolean; - doc_values?: boolean; - enabled?: boolean; - format?: string; - ignore_above?: number; - multi_fields?: MultiField[]; - index?: boolean; - path?: string; - scaling_factor?: number; - dynamic?: boolean | 'strict'; - properties?: Record; - }; -} diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts deleted file mode 100644 index 59945b23124c6..0000000000000 --- a/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { loggerMock } from '@kbn/logging-mocks'; - -import { installWithTimeout } from './install_with_timeout'; -import { ReplaySubject, type Subject } from 'rxjs'; - -const logger = loggerMock.create(); - -describe('installWithTimeout', () => { - let pluginStop$: Subject; - - beforeEach(() => { - jest.resetAllMocks(); - pluginStop$ = new ReplaySubject(1); - }); - - it(`should call installFn`, async () => { - const installFn = jest.fn(); - await installWithTimeout({ - installFn, - pluginStop$, - timeoutMs: 10, - }); - expect(installFn).toHaveBeenCalled(); - }); - - it(`should short-circuit installFn if it exceeds configured timeout`, async () => { - await expect(() => - installWithTimeout({ - installFn: async () => { - await new Promise((r) => setTimeout(r, 20)); - }, - pluginStop$, - timeoutMs: 10, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Failure during installation. Timeout: it took more than 10ms"` - ); - }); - - it(`should short-circuit installFn if pluginStop$ signal is received`, async () => { - pluginStop$.next(); - await expect(() => - installWithTimeout({ - installFn: async () => { - await new Promise((r) => setTimeout(r, 5)); - logger.info(`running`); - }, - pluginStop$, - timeoutMs: 10, - }) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Server is stopping; must stop all async operations"` - ); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.ts deleted file mode 100644 index 7995fed5152ad..0000000000000 --- a/packages/kbn-data-stream-adapter/src/install_with_timeout.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { firstValueFrom, type Observable } from 'rxjs'; - -const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes - -interface InstallWithTimeoutOpts { - description?: string; - installFn: () => Promise; - pluginStop$: Observable; - timeoutMs?: number; -} - -export class InstallShutdownError extends Error { - constructor() { - super('Server is stopping; must stop all async operations'); - Object.setPrototypeOf(this, InstallShutdownError.prototype); - } -} - -export const installWithTimeout = async ({ - description, - installFn, - pluginStop$, - timeoutMs = INSTALLATION_TIMEOUT, -}: InstallWithTimeoutOpts): Promise => { - try { - let timeoutId: NodeJS.Timeout; - const install = async (): Promise => { - await installFn(); - if (timeoutId) { - clearTimeout(timeoutId); - } - }; - - const throwTimeoutException = (): Promise => { - return new Promise((_, reject) => { - timeoutId = setTimeout(() => { - const msg = `Timeout: it took more than ${timeoutMs}ms`; - reject(new Error(msg)); - }, timeoutMs); - - firstValueFrom(pluginStop$).then(() => { - clearTimeout(timeoutId); - reject(new InstallShutdownError()); - }); - }); - }; - - await Promise.race([install(), throwTimeoutException()]); - } catch (e) { - if (e instanceof InstallShutdownError) { - throw e; - } else { - const reason = e?.message || 'Unknown reason'; - throw new Error( - `Failure during installation${description ? ` of ${description}` : ''}. ${reason}` - ); - } - } -}; diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts deleted file mode 100644 index e53eb7704a06a..0000000000000 --- a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { getIndexTemplate, getComponentTemplate } from './resource_installer_utils'; - -describe('getIndexTemplate', () => { - const defaultParams = { - name: 'indexTemplateName', - kibanaVersion: '8.12.1', - indexPatterns: ['indexPattern1', 'indexPattern2'], - componentTemplateRefs: ['template1', 'template2'], - totalFieldsLimit: 2500, - }; - - it('should create index template with given parameters and defaults', () => { - const indexTemplate = getIndexTemplate(defaultParams); - - expect(indexTemplate).toEqual({ - name: defaultParams.name, - body: { - data_stream: { hidden: true }, - index_patterns: defaultParams.indexPatterns, - composed_of: defaultParams.componentTemplateRefs, - template: { - settings: { - hidden: true, - auto_expand_replicas: '0-1', - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, - }, - mappings: { - dynamic: false, - _meta: { - kibana: { - version: defaultParams.kibanaVersion, - }, - managed: true, - namespace: 'default', - }, - }, - }, - _meta: { - kibana: { - version: defaultParams.kibanaVersion, - }, - managed: true, - namespace: 'default', - }, - priority: 7, - }, - }); - }); - - it('should create not hidden index template', () => { - const { body } = getIndexTemplate({ ...defaultParams, hidden: false }); - expect(body?.data_stream?.hidden).toEqual(false); - expect(body?.template?.settings?.hidden).toEqual(false); - }); - - it('should create index template with custom namespace', () => { - const { body } = getIndexTemplate({ ...defaultParams, namespace: 'custom-namespace' }); - expect(body?._meta?.namespace).toEqual('custom-namespace'); - expect(body?.priority).toEqual(16); - }); - - it('should create index template with template overrides', () => { - const { body } = getIndexTemplate({ - ...defaultParams, - template: { - settings: { - number_of_shards: 1, - }, - mappings: { - dynamic: true, - }, - lifecycle: { - data_retention: '30d', - }, - }, - }); - - expect(body?.template?.settings).toEqual({ - hidden: true, - auto_expand_replicas: '0-1', - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, - number_of_shards: 1, - }); - - expect(body?.template?.mappings).toEqual({ - dynamic: true, - _meta: { - kibana: { - version: defaultParams.kibanaVersion, - }, - managed: true, - namespace: 'default', - }, - }); - - expect(body?.template?.lifecycle).toEqual({ - data_retention: '30d', - }); - }); -}); - -describe('getComponentTemplate', () => { - const defaultParams = { - name: 'componentTemplateName', - kibanaVersion: '8.12.1', - fieldMap: { - field1: { type: 'text', required: true }, - field2: { type: 'keyword', required: false }, - }, - }; - - it('should create component template with given parameters and defaults', () => { - const componentTemplate = getComponentTemplate(defaultParams); - - expect(componentTemplate).toEqual({ - name: defaultParams.name, - _meta: { - managed: true, - }, - template: { - settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': 1500, - }, - mappings: { - dynamic: 'strict', - properties: { - field1: { - type: 'text', - }, - field2: { - type: 'keyword', - }, - }, - }, - }, - }); - }); - - it('should create component template with custom settings', () => { - const { template } = getComponentTemplate({ - ...defaultParams, - settings: { - number_of_shards: 1, - number_of_replicas: 1, - }, - }); - - expect(template.settings).toEqual({ - number_of_shards: 1, - number_of_replicas: 1, - 'index.mapping.total_fields.limit': 1500, - }); - }); - - it('should create component template with custom dynamic', () => { - const { template } = getComponentTemplate({ ...defaultParams, dynamic: true }); - expect(template.mappings?.dynamic).toEqual(true); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts deleted file mode 100644 index 456be9ad8e86f..0000000000000 --- a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { - IndicesPutIndexTemplateRequest, - Metadata, -} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { - ClusterPutComponentTemplateRequest, - IndicesIndexSettings, - IndicesPutIndexTemplateIndexTemplateMapping, -} from '@elastic/elasticsearch/lib/api/types'; -import type { FieldMap } from './field_maps/types'; -import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; - -interface GetComponentTemplateOpts { - name: string; - fieldMap: FieldMap; - settings?: IndicesIndexSettings; - dynamic?: 'strict' | boolean; -} - -export const getComponentTemplate = ({ - name, - fieldMap, - settings, - dynamic = 'strict', -}: GetComponentTemplateOpts): ClusterPutComponentTemplateRequest => ({ - name, - _meta: { - managed: true, - }, - template: { - settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': - Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, - ...settings, - }, - mappings: mappingFromFieldMap(fieldMap, dynamic), - }, -}); - -interface GetIndexTemplateOpts { - name: string; - indexPatterns: string[]; - kibanaVersion: string; - totalFieldsLimit: number; - componentTemplateRefs?: string[]; - namespace?: string; - template?: IndicesPutIndexTemplateIndexTemplateMapping; - hidden?: boolean; -} - -export const getIndexTemplate = ({ - name, - indexPatterns, - kibanaVersion, - totalFieldsLimit, - componentTemplateRefs, - namespace = 'default', - template = {}, - hidden = true, -}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => { - const indexMetadata: Metadata = { - kibana: { - version: kibanaVersion, - }, - managed: true, - namespace, - }; - - return { - name, - body: { - data_stream: { hidden }, - index_patterns: indexPatterns, - composed_of: componentTemplateRefs, - template: { - ...template, - settings: { - hidden, - auto_expand_replicas: '0-1', - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': totalFieldsLimit, - ...template.settings, - }, - mappings: { - dynamic: false, - _meta: indexMetadata, - ...template.mappings, - }, - }, - _meta: indexMetadata, - - // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace - // then newly created indices will use the matching template with the *longest* namespace - priority: namespace.length, - }, - }; -}; diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts deleted file mode 100644 index f7d6cca8c5a07..0000000000000 --- a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { errors as EsErrors, type DiagnosticResult } from '@elastic/elasticsearch'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; - -const mockLogger = loggingSystemMock.createLogger(); - -// mock setTimeout to avoid waiting in tests and prevent test flakiness -global.setTimeout = jest.fn((cb) => jest.fn(cb())) as unknown as typeof global.setTimeout; - -describe('retryTransientEsErrors', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it.each([ - { error: new EsErrors.ConnectionError('test error'), errorType: 'ConnectionError' }, - { - error: new EsErrors.NoLivingConnectionsError('test error', {} as DiagnosticResult), - errorType: 'NoLivingConnectionsError', - }, - { error: new EsErrors.TimeoutError('test error'), errorType: 'TimeoutError' }, - { - error: new EsErrors.ResponseError({ statusCode: 503 } as DiagnosticResult), - errorType: 'ResponseError (Unavailable)', - }, - { - error: new EsErrors.ResponseError({ statusCode: 408 } as DiagnosticResult), - errorType: 'ResponseError (RequestTimeout)', - }, - { - error: new EsErrors.ResponseError({ statusCode: 410 } as DiagnosticResult), - errorType: 'ResponseError (Gone)', - }, - ])('should retry $errorType', async ({ error }) => { - const mockFn = jest.fn(); - mockFn.mockRejectedValueOnce(error); - mockFn.mockResolvedValueOnce('success'); - - const result = await retryTransientEsErrors(mockFn, { logger: mockLogger }); - - expect(result).toEqual('success'); - expect(mockFn).toHaveBeenCalledTimes(2); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); - expect(mockLogger.error).not.toHaveBeenCalled(); - }); - - it('should throw non-transient errors', async () => { - const error = new EsErrors.ResponseError({ statusCode: 429 } as DiagnosticResult); - const mockFn = jest.fn(); - mockFn.mockRejectedValueOnce(error); - - await expect(retryTransientEsErrors(mockFn, { logger: mockLogger })).rejects.toEqual(error); - - expect(mockFn).toHaveBeenCalledTimes(1); - expect(mockLogger.warn).not.toHaveBeenCalled(); - }); - - it('should throw if max retries exceeded', async () => { - const error = new EsErrors.ConnectionError('test error'); - const mockFn = jest.fn(); - mockFn.mockRejectedValueOnce(error); - mockFn.mockRejectedValueOnce(error); - - await expect( - retryTransientEsErrors(mockFn, { logger: mockLogger, attempt: 2 }) - ).rejects.toEqual(error); - - expect(mockFn).toHaveBeenCalledTimes(2); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); - }); -}); diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts deleted file mode 100644 index 3b436298e5c8d..0000000000000 --- a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { Logger } from '@kbn/core/server'; -import { errors as EsErrors } from '@elastic/elasticsearch'; - -const MAX_ATTEMPTS = 3; - -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: Error) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && - e?.statusCode && - retryResponseStatuses.includes(e.statusCode)); - -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -export const retryTransientEsErrors = async ( - esCall: () => Promise, - { logger, attempt = 0 }: { logger: Logger; attempt?: number } -): Promise => { - try { - return await esCall(); - } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { - const retryCount = attempt + 1; - const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... - - logger.warn( - `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ - e.stack - }` - ); - - // delay with some randomness - await delay(retryDelaySec * 1000 * Math.random()); - return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); - } - - throw e; - } -}; diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json deleted file mode 100644 index f09d2b4354d02..0000000000000 --- a/packages/kbn-data-stream-adapter/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "target/types", - "types": [ - "jest", - "node", - "react", - "@emotion/react/types/css-prop", - "@testing-library/jest-dom", - "@testing-library/react" - ] - }, - "include": ["**/*.ts", "**/*.tsx"], - "kbn_references": [ - "@kbn/core", - "@kbn/std", - "@kbn/ecs", - "@kbn/alerts-as-data-utils", - "@kbn/safer-lodash-set", - "@kbn/logging-mocks", - ], - "exclude": ["target/**/*"] -} From aab5c121ee3137cc1c3cb98a3de9194bcf9adf97 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 24 Jan 2024 15:42:07 -0800 Subject: [PATCH 042/141] fixed start fetch error --- .../impl/assistant/index.tsx | 21 ++++--- .../impl/assistant/use_conversation/index.tsx | 22 ++++--- .../connector_selector_inline.tsx | 10 ++-- .../connectorland/connector_setup/index.tsx | 59 +++++++------------ .../use_conversation_store/index.tsx | 6 +- 5 files changed, 57 insertions(+), 61 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 8bd13f9d04a96..72af518c3287d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -55,6 +55,7 @@ import { useFetchCurrentUserConversations, } from './api/conversations/use_fetch_current_user_conversations'; import { Conversation } from '../assistant_context/types'; +import { clearPresentationData } from '../connectorland/connector_setup/helpers'; export interface Props { conversationId?: string; @@ -134,7 +135,7 @@ const AssistantComponent: React.FC = ({ isLoading, isError, refetch, - } = useFetchCurrentUserConversations(onFetchedConversations); + } = useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); useEffect(() => { if (!isLoading && !isError) { @@ -245,12 +246,6 @@ const AssistantComponent: React.FC = ({ setLastConversationId, ]); - const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ - conversation: blockBotConversation, - conversations, - setConversations, - }); - const [promptTextPreview, setPromptTextPreview] = useState(''); const [autoPopulatedOnce, setAutoPopulatedOnce] = useState(false); const [userPrompt, setUserPrompt] = useState(null); @@ -348,6 +343,18 @@ const AssistantComponent: React.FC = ({ [allSystemPrompts, conversations, refetchCurrentConversation, refetchResults] ); + const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ + conversation: blockBotConversation, + onConversationUpdate: handleOnConversationSelected, + onSetupComplete: () => { + console.log('clear') + setConversations({ + ...conversations, + [currentConversation.id]: clearPresentationData(currentConversation), + }); + }, + }); + const handleOnConversationDeleted = useCallback( async (cId: string) => { setTimeout(() => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 8395b1fd95428..83dcad0f51441 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -48,9 +48,7 @@ interface CreateConversationProps { } interface SetApiConfigProps { - conversationId: string; - isDefault?: boolean; - title: string; + conversation: Conversation; apiConfig: Conversation['apiConfig']; } @@ -69,10 +67,8 @@ interface UseConversation { deleteConversation: (conversationId: string) => void; removeLastMessage: (conversationId: string) => Promise; setApiConfig: ({ - conversationId, + conversation, apiConfig, - title, - isDefault, }: SetApiConfigProps) => Promise>; createConversation: (conversation: Conversation) => Promise; getConversation: (conversationId: string) => Promise; @@ -275,22 +271,24 @@ export const useConversation = (): UseConversation => { * Create/Update the apiConfig for a given conversationId */ const setApiConfig = useCallback( - async ({ conversationId, apiConfig, title, isDefault }: SetApiConfigProps) => { - if (title === conversationId) { + async ({ conversation, apiConfig }: SetApiConfigProps) => { + if (conversation.title === conversation.id) { return createConversationApi({ http, conversation: { apiConfig, - title, - isDefault, + title: conversation.title, + replacements: conversation.replacements, + excludeFromLastConversationStorage: conversation.excludeFromLastConversationStorage, + isDefault: conversation.isDefault, id: '', - messages: [], + messages: conversation.messages ?? [], }, }); } else { return updateConversationApi({ http, - conversationId, + conversationId: conversation.id, apiConfig, }); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx index 520240cfb1581..4ea89ef252348 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx @@ -9,6 +9,7 @@ import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui import React, { useCallback, useMemo, useState } from 'react'; import { css } from '@emotion/css'; +import { isHttpFetchError } from '@kbn/core-http-browser'; import { AIConnector, ConnectorSelector } from '../connector_selector'; import { Conversation } from '../../..'; import { useLoadConnectors } from '../use_load_connectors'; @@ -107,9 +108,7 @@ export const ConnectorSelectorInline: React.FC = React.memo( if (selectedConversation != null) { const conversation = await setApiConfig({ - conversationId: selectedConversation.id, - title: selectedConversation.title, - isDefault: selectedConversation.isDefault, + conversation: selectedConversation, apiConfig: { ...selectedConversation.apiConfig, connectorId, @@ -119,7 +118,10 @@ export const ConnectorSelectorInline: React.FC = React.memo( model: model ?? config?.defaultModel, }, }); - onConnectorSelected(conversation as Conversation); + + if (!isHttpFetchError(conversation)) { + onConnectorSelected(conversation); + } } }, [selectedConversation, setApiConfig, onConnectorSelected] diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 966cc6400c7bd..e2925c5a80b85 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -13,6 +13,7 @@ import styled from 'styled-components'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public/common/constants'; import { ActionType } from '@kbn/triggers-actions-ui-plugin/public'; +import { isHttpFetchError } from '@kbn/core-http-browser'; import { AddConnectorModal } from '../add_connector_modal'; import { WELCOME_CONVERSATION } from '../../assistant/use_conversation/sample_conversations'; import { Conversation, Message } from '../../..'; @@ -38,15 +39,13 @@ const SkipEuiText = styled(EuiText)` export interface ConnectorSetupProps { conversation?: Conversation; onSetupComplete?: () => void; - conversations: Record; - setConversations: React.Dispatch>>; + onConversationUpdate: (cId: string, cTitle?: string) => Promise; } export const useConnectorSetup = ({ conversation = WELCOME_CONVERSATION, onSetupComplete, - conversations, - setConversations, + onConversationUpdate, }: ConnectorSetupProps): { comments: EuiCommentProps[]; prompt: React.ReactElement; @@ -104,8 +103,8 @@ export const useConnectorSetup = ({ setShowAddConnectorButton(true); bottomRef.current?.scrollIntoView({ block: 'end' }); onSetupComplete?.(); - setConversations({ ...conversations, [conversation.id]: clearPresentationData(conversation) }); - }, [conversation, conversations, onSetupComplete, setConversations]); + // setConversations({ ...conversations, [conversation.id]: clearPresentationData(conversation) }); + }, [onSetupComplete]); // Show button to add connector after last message has finished streaming const handleSkipSetup = useCallback(() => { @@ -180,10 +179,8 @@ export const useConnectorSetup = ({ const connectorTypeTitle = getActionTypeTitle(actionTypeRegistry.get(connector.actionTypeId)); // persist only the active conversation - await setApiConfig({ - conversationId: conversation.id, - title: conversation.title, - isDefault: conversation.isDefault, + const updatedConversation = await setApiConfig({ + conversation, apiConfig: { ...conversation.apiConfig, connectorId: connector.id, @@ -192,41 +189,29 @@ export const useConnectorSetup = ({ model: config?.defaultModel, }, }); - setConversations( - Object.values(conversations).reduce((res, c) => { - res[c.id] = { - ...c, - apiConfig: { - ...c.apiConfig, - connectorId: connector.id, - connectorTypeTitle, - provider: config?.apiProvider, - model: config?.defaultModel, - }, - }; - return res; - }, {} as Record) - ); - refetchConnectors?.(); - setIsConnectorModalVisible(false); - await appendMessage({ - conversationId: conversation.id, - message: { - role: 'assistant', - content: i18n.CONNECTOR_SETUP_COMPLETE, - timestamp: new Date().toLocaleString(), - }, - }); + if (!isHttpFetchError(updatedConversation)) { + onConversationUpdate(updatedConversation.id, updatedConversation.title); + + refetchConnectors?.(); + setIsConnectorModalVisible(false); + await appendMessage({ + conversationId: updatedConversation.id, + message: { + role: 'assistant', + content: i18n.CONNECTOR_SETUP_COMPLETE, + timestamp: new Date().toLocaleString(), + }, + }); + } }, [ actionTypeRegistry, appendMessage, conversation, - conversations, + onConversationUpdate, refetchConnectors, setApiConfig, - setConversations, ] ); diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx index 5676759529a3d..ab88f3bd33745 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.tsx @@ -14,9 +14,13 @@ import type { FetchConversationsResponse } from '@kbn/elastic-assistant/impl/ass import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { useLinkAuthorized } from '../../common/links'; import { SecurityPageName } from '../../../common'; +import { useKibana } from '../../common/lib/kibana'; export const useConversationStore = (): Record => { const [conversations, setConversations] = useState>({}); + const { + services: { http }, + } = useKibana(); const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); const baseConversations = useMemo( () => @@ -53,7 +57,7 @@ export const useConversationStore = (): Record => { data: conversationsData, isLoading, isError, - } = useFetchCurrentUserConversations(onFetchedConversations); + } = useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); useEffect(() => { if (!isLoading && !isError) { From 35ee2afd670d1bd9d5c7f16788a311ee877a74fe Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 24 Jan 2024 17:53:12 -0800 Subject: [PATCH 043/141] fixed starting conversation failures --- .../kbn-elastic-assistant/impl/assistant/index.tsx | 11 +++++++++-- .../impl/connectorland/connector_setup/index.tsx | 3 +-- .../assistant/use_assistant_telemetry/index.tsx | 8 ++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 72af518c3287d..dadf5387522bc 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -347,7 +347,6 @@ const AssistantComponent: React.FC = ({ conversation: blockBotConversation, onConversationUpdate: handleOnConversationSelected, onSetupComplete: () => { - console.log('clear') setConversations({ ...conversations, [currentConversation.id]: clearPresentationData(currentConversation), @@ -587,7 +586,15 @@ const AssistantComponent: React.FC = ({ refetchConversationsState={async () => { const refetchedConversations = await refetchResults(); if (refetchedConversations && refetchedConversations[selectedConversationId]) { - await refetchCurrentConversation(); + setCurrentConversation(refetchedConversations[selectedConversationId]); + } else if (refetchedConversations) { + const createdSelectedConversation = Object.values(refetchedConversations).find( + (c) => c.title === selectedConversationId + ); + if (createdSelectedConversation) { + setCurrentConversation(createdSelectedConversation); + setSelectedConversationId(createdSelectedConversation.id); + } } }} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index e2925c5a80b85..3b59353ea8227 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -21,7 +21,7 @@ import { useLoadActionTypes } from '../use_load_action_types'; import { StreamingText } from '../../assistant/streaming_text'; import { ConnectorButton } from '../connector_button'; import { useConversation } from '../../assistant/use_conversation'; -import { clearPresentationData, conversationHasNoPresentationData } from './helpers'; +import { conversationHasNoPresentationData } from './helpers'; import * as i18n from '../translations'; import { useAssistantContext } from '../../assistant_context'; import { useLoadConnectors } from '../use_load_connectors'; @@ -103,7 +103,6 @@ export const useConnectorSetup = ({ setShowAddConnectorButton(true); bottomRef.current?.scrollIntoView({ block: 'end' }); onSetupComplete?.(); - // setConversations({ ...conversations, [conversation.id]: clearPresentationData(conversation) }); }, [onSetupComplete]); // Show button to add connector after last message has finished streaming diff --git a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx index e3ca26b31f24a..95d8cc16b5d38 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.tsx @@ -9,19 +9,23 @@ import type { Conversation } from '@kbn/elastic-assistant'; import { getConversationById, type AssistantTelemetry } from '@kbn/elastic-assistant'; import { useCallback } from 'react'; import { useKibana } from '../../common/lib/kibana'; +import { useBaseConversations } from '../use_conversation_store'; export const useAssistantTelemetry = (): AssistantTelemetry => { const { services: { telemetry, http }, } = useKibana(); + const baseConversations = useBaseConversations(); const getAnonymizedConversationId = useCallback( async (id) => { - const conversation = await getConversationById({ http, id }); + const conversation = baseConversations[id] + ? baseConversations[id] + : await getConversationById({ http, id }); const convo = (conversation as Conversation) ?? { isDefault: false }; return convo.isDefault ? id : 'Custom'; }, - [http] + [baseConversations, http] ); const reportTelemetry = useCallback( From 36286968b2c7b73a87d4859e69541f6fb0c86deb Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 24 Jan 2024 18:39:44 -0800 Subject: [PATCH 044/141] tests fixes --- .../group5/dot_kibana_split.test.ts | 2 ++ ..._fetch_current_user_conversations.test.tsx | 33 ++++++++++++------- .../assistant/use_conversation/index.test.tsx | 4 +-- .../connector_selector_inline.test.tsx | 8 +++-- .../connector_setup/index.test.tsx | 13 +++----- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 9f9c3a3c7bd58..5ecb474821ede 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -203,6 +203,8 @@ describe('split .kibana index into multiple system indices', () => { "connector_token", "core-usage-stats", "csp-rule-template", + "elastic-ai-assistant-anonimization-fields", + "elastic-ai-assistant-prompts", "endpoint:user-artifact-manifest", "enterprise_search_telemetry", "epm-packages", diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx index 6af11ce6cef9b..06aaa41a4b118 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { renderHook } from '@testing-library/react-hooks'; +import { act, renderHook } from '@testing-library/react-hooks'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import type { ReactNode } from 'react'; @@ -33,19 +33,30 @@ const createWrapper = () => { }; describe('useFetchCurrentUserConversations', () => { - it(`should make http request to fetch conversations`, () => { + it(`should make http request to fetch conversations`, async () => { renderHook(() => useFetchCurrentUserConversations(defaultProps), { wrapper: createWrapper(), }); - expect(defaultProps.http.fetch).toHaveBeenCalledWith( - '/api/elastic_assistant/conversations/current_user/_find', - { - method: 'GET', - version: '2023-10-31', - signal: undefined, - } - ); - expect(onFetch).toHaveBeenCalled(); + await act(async () => { + const { waitForNextUpdate } = renderHook(() => + useFetchCurrentUserConversations(defaultProps) + ); + await waitForNextUpdate(); + expect(defaultProps.http.fetch).toHaveBeenCalledWith( + '/api/elastic_assistant/conversations/current_user/_find', + { + method: 'GET', + query: { + page: 1, + perPage: 100, + }, + version: '2023-10-31', + signal: undefined, + } + ); + + expect(onFetch).toHaveBeenCalled(); + }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index f3c3c29d06cb5..24e4b8f1fcba6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -18,6 +18,7 @@ import { updateConversationApi, } from '../api/conversations'; import { httpServiceMock } from '@kbn/core/public/mocks'; +import { WELCOME_CONVERSATION } from './sample_conversations'; jest.mock('../api/conversations', () => { const actual = jest.requireActual('../api/conversations'); @@ -155,9 +156,8 @@ describe('useConversation', () => { await waitForNextUpdate(); await result.current.setApiConfig({ - conversationId: welcomeConvo.id, + conversation: WELCOME_CONVERSATION, apiConfig: mockConvo.apiConfig, - title: welcomeConvo.title, }); expect(updateConversationApiMock).toHaveBeenCalledWith({ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx index a5b6c79f89d51..19b5f24e221c3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx @@ -141,8 +141,12 @@ describe('ConnectorSelectorInline', () => { model: undefined, provider: 'OpenAI', }, - conversationId: 'conversation_id', - title: 'conversation_id', + conversation: { + apiConfig: {}, + id: 'conversation_id', + messages: [], + title: 'conversation_id', + }, }); }); it('On connector change to add new connector, onchange event does nothing', () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx index 4b1ab2ec88038..1bf6b6bc6ff2b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx @@ -9,20 +9,17 @@ import React from 'react'; import { useConnectorSetup } from '.'; import { act, renderHook } from '@testing-library/react-hooks'; import { fireEvent, render } from '@testing-library/react'; -import { alertConvo, welcomeConvo } from '../../mock/conversation'; +import { welcomeConvo } from '../../mock/conversation'; import { TestProviders } from '../../mock/test_providers/test_providers'; import { EuiCommentList } from '@elastic/eui'; const onSetupComplete = jest.fn(); -const setConversations = jest.fn(); +const onConversationUpdate = jest.fn(); + const defaultProps = { conversation: welcomeConvo, onSetupComplete, - setConversations, - conversations: { - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - }, + onConversationUpdate, }; const newConnector = { actionTypeId: '.gen-ai', name: 'cool name' }; jest.mock('../add_connector_modal', () => ({ @@ -141,7 +138,7 @@ describe('useConnectorSetup', () => { expect(clearTimeout).toHaveBeenCalled(); expect(onSetupComplete).toHaveBeenCalled(); - expect(setConversations).toHaveBeenCalled(); + expect(onConversationUpdate).toHaveBeenCalled(); }); }); }); From d3b6d6e819223d26a5a4152dcb350cb7753cee62 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 24 Jan 2024 21:42:40 -0800 Subject: [PATCH 045/141] - --- .../system_prompt/index.test.tsx | 1 + .../assistant/use_conversation/index.test.tsx | 4 +- ...reate_resource_installation_helper.test.ts | 207 +++++---------- .../conversations_data_writer.test.ts | 247 +++--------------- .../create_conversation.test.ts | 94 +++++-- .../update_conversation.test.ts | 94 +++++-- 6 files changed, 242 insertions(+), 405 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx index 519f7815c9640..3323f05b062fe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx @@ -375,6 +375,7 @@ describe('SystemPrompt', () => { apiConfig: { defaultSystemPromptId: undefined, }, + title: 'second', messages: [], }; const localMockConversations: Record = { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index 24e4b8f1fcba6..8d4d1abb31406 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -61,7 +61,7 @@ describe('useConversation', () => { let httpMock: ReturnType; beforeEach(() => { - httpMock = httpServiceMock.createSetupContract({ basePath: '/test' }); + httpMock = httpServiceMock.createSetupContract(); jest.clearAllMocks(); }); @@ -226,8 +226,8 @@ describe('useConversation', () => { }); expect(appendConversationMessagesApiMock).toHaveBeenCalledWith({ - http: httpMock, conversationId: mockConvo.id, + http: httpMock, messages: [ { ...anotherMessage, diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts index 4ac6d4cc69810..49a8bed94bca7 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts @@ -17,19 +17,39 @@ import { calculateDelay, getShouldRetry, } from './create_resource_installation_helper'; -import { retryUntil } from './test_utils'; -const logger: ReturnType = - loggingSystemMock.createLogger(); +const RETRY_UNTIL_DEFAULT_COUNT = 20; +const RETRY_UNTIL_DEFAULT_WAIT = 100; // milliseconds +type RetryableFunction = () => Promise; -const initFn = async (context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { - logger.info(`${context.context}_${namespace}`); -}; +export const retryUntil = async ( + label: string, + fn: RetryableFunction, + count: number = RETRY_UNTIL_DEFAULT_COUNT, + wait: number = RETRY_UNTIL_DEFAULT_WAIT +): Promise => { + await delay(wait); + while (count > 0) { + // eslint-disable-next-line no-param-reassign + count--; + + if (await fn()) return true; + + // eslint-disable-next-line no-console + console.log(`attempt failed waiting for "${label}", attempts left: ${count}`); + + if (count === 0) return false; + await delay(wait); + } -const initFnWithError = async (context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { - throw new Error('no go'); + return false; }; +const delay = async (millis: number) => new Promise((resolve) => setTimeout(resolve, millis)); + +const logger: ReturnType = + loggingSystemMock.createLogger(); + const getCommonInitPromise = async ( resolution: boolean, timeoutMs: number = 1, @@ -50,7 +70,7 @@ const getContextInitialized = async ( context: string = 'test1', namespace: string = DEFAULT_NAMESPACE_STRING ) => { - const { result } = await helper.getInitializedResources(context, namespace); + const { result } = await helper.getInitializedResources(namespace); return result; }; @@ -63,119 +83,70 @@ describe('createResourceInstallationHelper', () => { const helper = createResourceInstallationHelper( logger, getCommonInitPromise(true, 100), - initFn + (namespace: string) => { + return Promise.resolve(); + } ); // Add two contexts that need to be initialized - helper.add({ - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); - helper.add({ - context: 'test2', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); + helper.add('test1'); + helper.add('test2'); await retryUntil('init fns run', async () => logger.info.mock.calls.length === 3); expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); expect(logger.info).toHaveBeenNthCalledWith(2, 'test1_default'); expect(logger.info).toHaveBeenNthCalledWith(3, 'test2_default'); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: true, }); - expect(await helper.getInitializedResources('test2', DEFAULT_NAMESPACE_STRING)).toEqual({ - result: true, - }); - }); - - test(`should return false if context is unrecognized`, async () => { - const helper = createResourceInstallationHelper( - logger, - getCommonInitPromise(true, 100), - initFn - ); - - helper.add({ - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); - - await retryUntil('init fns run', async () => logger.info.mock.calls.length === 2); - - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: true, }); - expect(await helper.getInitializedResources('test2', DEFAULT_NAMESPACE_STRING)).toEqual({ - result: false, - error: `Unrecognized context test2_default`, - }); }); test(`should log and return false if common init function returns false`, async () => { const helper = createResourceInstallationHelper( logger, getCommonInitPromise(false, 100), - initFn + (namespace: string) => { + return Promise.resolve(); + } ); - helper.add({ - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); + helper.add('test1'); await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); expect(logger.warn).toHaveBeenCalledWith( `Common resources were not initialized, cannot initialize context for test1` ); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, error: `error initializing`, }); }); test(`should log and return false if common init function throws error`, async () => { - const helper = createResourceInstallationHelper(logger, getCommonInitPromise(true, -1), initFn); - - helper.add({ - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); - - await retryUntil( - 'common init fns run', - async () => (await getContextInitialized(helper)) === false - ); - - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - fail`); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ - result: false, - error: `fail`, - }); - }); - - test(`should log and return false if context init function throws error`, async () => { const helper = createResourceInstallationHelper( logger, - getCommonInitPromise(true, 100), - initFnWithError + getCommonInitPromise(true, -1), + (namespace: string) => { + return Promise.resolve(); + } ); - helper.add({ - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }); + helper.add('test1'); await retryUntil( - 'context init fns run', + 'common init fns run', async () => (await getContextInitialized(helper)) === false ); - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - no go`); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - fail`); + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, - error: `no go`, + error: `fail`, }); }); @@ -187,73 +158,28 @@ describe('createResourceInstallationHelper', () => { const helper = createResourceInstallationHelper( logger, getCommonInitPromise(false, 100), - initFn + (namespace: string) => { + return Promise.resolve(); + } ); - helper.add(context); + helper.add(); await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); expect(logger.warn).toHaveBeenCalledWith( `Common resources were not initialized, cannot initialize context for test1` ); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, error: `error initializing`, }); - helper.retry(context, undefined, getCommonInitPromise(true, 100, 'after retry')); + helper.retry(undefined, getCommonInitPromise(true, 100, 'after retry')); await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 2); expect(logger.info).toHaveBeenCalledWith(`commonInitPromise resolved - after retry`); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ - result: true, - }); - }); - - test(`should retry context init function`, async () => { - const initFnErrorOnce = jest - .fn() - .mockImplementationOnce(() => { - throw new Error('first error'); - }) - .mockImplementation((context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { - logger.info(`${context.context}_${namespace} successfully retried`); - }); - const context = { - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }; - const helper = createResourceInstallationHelper( - logger, - getCommonInitPromise(true, 100), - initFnErrorOnce - ); - - helper.add(context); - - await retryUntil( - 'context init fns run', - async () => (await getContextInitialized(helper)) === false - ); - - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - first error`); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ - result: false, - error: `first error`, - }); - - helper.retry(context, undefined); - - await retryUntil('init fns retried', async () => logger.info.mock.calls.length === 3); - - expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); - expect(logger.info).toHaveBeenNthCalledWith( - 2, - `Retrying resource initialization for context "test1"` - ); - expect(logger.info).toHaveBeenNthCalledWith(3, 'test1_default successfully retried'); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: true, }); }); @@ -267,20 +193,17 @@ describe('createResourceInstallationHelper', () => { .mockImplementationOnce(() => { throw new Error('second error'); }) - .mockImplementation((context: IRuleTypeAlerts, namespace: string, timeoutMs?: number) => { - logger.info(`${context.context}_${namespace} successfully retried`); + .mockImplementation((namespace: string, timeoutMs?: number) => { + logger.info(`${namespace} successfully retried`); }); - const context = { - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }; + const helper = createResourceInstallationHelper( logger, getCommonInitPromise(true, 100), initFnErrorOnce ); - helper.add(context); + helper.add(); await retryUntil( 'context init fns run', @@ -288,7 +211,7 @@ describe('createResourceInstallationHelper', () => { ); expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - first error`); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, error: `first error`, }); @@ -296,9 +219,9 @@ describe('createResourceInstallationHelper', () => { logger.info.mockClear(); logger.error.mockClear(); - helper.retry(context, undefined); + helper.retry(undefined); await new Promise((r) => setTimeout(r, 10)); - helper.retry(context, undefined); + helper.retry(undefined); await retryUntil('init fns retried', async () => { return logger.error.mock.calls.length === 1; @@ -308,7 +231,7 @@ describe('createResourceInstallationHelper', () => { // the second retry is throttled so this is never called expect(logger.info).not.toHaveBeenCalledWith('test1_default successfully retried'); - expect(await helper.getInitializedResources('test1', DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, error: 'second error', }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts index 271d5511e6b75..644a895693dcb 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts @@ -7,29 +7,31 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; -import { RiskEngineDataWriter } from './risk_engine_data_writer'; -import { riskScoreServiceMock } from './risk_score_service.mock'; +import { ConversationDataWriter } from './conversations_data_writer'; -describe('RiskEngineDataWriter', () => { +describe('ConversationDataWriter', () => { describe('#bulk', () => { - let writer: RiskEngineDataWriter; + let writer: ConversationDataWriter; let esClientMock: ElasticsearchClient; let loggerMock: Logger; beforeEach(() => { esClientMock = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; loggerMock = loggingSystemMock.createLogger(); - writer = new RiskEngineDataWriter({ + writer = new ConversationDataWriter({ esClient: esClientMock, logger: loggerMock, - index: 'risk-score.risk-score-default', - namespace: 'default', + index: 'conversations-default', + spaceId: 'default', + user: { name: 'test' }, }); }); - it('converts a list of host risk scores to an appropriate list of operations', async () => { + it('converts a list of conversations to an appropriate list of operations', async () => { await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore(), riskScoreServiceMock.createRiskScore()], + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; @@ -38,152 +40,29 @@ describe('RiskEngineDataWriter', () => { Array [ Object { "create": Object { - "_index": "risk-score.risk-score-default", + "_index": "conversations-default", }, }, Object { "@timestamp": "2023-02-15T00:15:19.231Z", - "host": Object { - "name": "hostname", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "host.name", - "id_value": "hostname", - "inputs": Array [], - "notes": Array [], - }, - }, - }, - Object { - "create": Object { - "_index": "risk-score.risk-score-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - "host": Object { - "name": "hostname", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "host.name", - "id_value": "hostname", - "inputs": Array [], - "notes": Array [], - }, - }, - }, - ] - `); - }); - - it('converts a list of user risk scores to an appropriate list of operations', async () => { - await writer.bulk({ - user: [ - riskScoreServiceMock.createRiskScore({ - id_field: 'user.name', - id_value: 'username_1', - }), - riskScoreServiceMock.createRiskScore({ - id_field: 'user.name', - id_value: 'username_2', - }), - ], - }); - - const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; - - expect(operations).toMatchInlineSnapshot(` - Array [ - Object { - "create": Object { - "_index": "risk-score.risk-score-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - "user": Object { - "name": "username_1", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "user.name", - "id_value": "username_1", - "inputs": Array [], - "notes": Array [], - }, - }, }, Object { "create": Object { - "_index": "risk-score.risk-score-default", + "_index": "conversations-default", }, }, Object { "@timestamp": "2023-02-15T00:15:19.231Z", - "user": Object { - "name": "username_2", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "user.name", - "id_value": "username_2", - "inputs": Array [], - "notes": Array [], - }, - }, }, ] `); }); - it('converts a list of mixed risk scores to an appropriate list of operations', async () => { + it('converts a list of mixed conversations operations to an appropriate list of operations', async () => { await writer.bulk({ - host: [ - riskScoreServiceMock.createRiskScore({ - id_field: 'host.name', - id_value: 'hostname_1', - }), - ], - user: [ - riskScoreServiceMock.createRiskScore({ - id_field: 'user.name', - id_value: 'username_1', - }), - riskScoreServiceMock.createRiskScore({ - id_field: 'user.name', - id_value: 'username_2', - }), - ], + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; @@ -192,81 +71,27 @@ describe('RiskEngineDataWriter', () => { Array [ Object { "create": Object { - "_index": "risk-score.risk-score-default", + "_index": "conversations-default", }, }, Object { "@timestamp": "2023-02-15T00:15:19.231Z", - "host": Object { - "name": "hostname_1", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "host.name", - "id_value": "hostname_1", - "inputs": Array [], - "notes": Array [], - }, - }, }, Object { - "create": Object { - "_index": "risk-score.risk-score-default", + "update": Object { + "_index": "conversations-default", }, }, Object { "@timestamp": "2023-02-15T00:15:19.231Z", - "user": Object { - "name": "username_1", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "user.name", - "id_value": "username_1", - "inputs": Array [], - "notes": Array [], - }, - }, }, Object { - "create": Object { - "_index": "risk-score.risk-score-default", + "delete": Object { + "_index": "conversations-default", }, }, Object { "@timestamp": "2023-02-15T00:15:19.231Z", - "user": Object { - "name": "username_2", - "risk": Object { - "calculated_level": "High", - "calculated_score": 149, - "calculated_score_norm": 85.332, - "category_1_count": 12, - "category_1_score": 85, - "category_2_count": 0, - "category_2_score": 0, - "criticality_level": "very_important", - "criticality_modifier": 2, - "id_field": "user.name", - "id_value": "username_2", - "inputs": Array [], - "notes": Array [], - }, - }, }, ] `); @@ -276,20 +101,24 @@ describe('RiskEngineDataWriter', () => { (esClientMock.bulk as jest.Mock).mockRejectedValue(new Error('something went wrong')); const { errors } = await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore()], + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); expect(errors).toEqual(['something went wrong']); }); - it('returns the time it took to write the risk scores', async () => { + it('returns the time it took to write the conversations', async () => { (esClientMock.bulk as jest.Mock).mockResolvedValue({ took: 123, items: [], }); const { took } = await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore()], + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); expect(took).toEqual(123); @@ -300,8 +129,10 @@ describe('RiskEngineDataWriter', () => { items: [{ create: { status: 201 } }, { create: { status: 200 } }], }); - const { docs_written: docsWritten } = await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore()], + const { docs_deleted: docsWritten } = await writer.bulk({ + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); expect(docsWritten).toEqual(2); @@ -319,8 +150,10 @@ describe('RiskEngineDataWriter', () => { }); it('returns the number of docs written', async () => { - const { docs_written: docsWritten } = await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore()], + const { docs_created: docsWritten } = await writer.bulk({ + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); expect(docsWritten).toEqual(1); @@ -328,14 +161,16 @@ describe('RiskEngineDataWriter', () => { it('returns the errors', async () => { const { errors } = await writer.bulk({ - host: [riskScoreServiceMock.createRiskScore()], + conversationsToCreate: [], + conversationsToUpdate: [], + conversationsToDelete: ['1'], }); expect(errors).toEqual(['something went wrong']); }); }); - describe('when there are no risk scores to write', () => { + describe('when there are no conversations to write', () => { it('returns an appropriate response', async () => { const response = await writer.bulk({}); expect(response).toEqual({ errors: [], docs_written: 0, took: 0 }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index f125d45c291ce..db5277e144512 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -6,8 +6,24 @@ */ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { createConversation } from './create_conversation'; +import { CreateMessageSchema, createConversation } from './create_conversation'; +import { ConversationCreateProps } from '../schemas/conversations/common_attributes.gen'; + +export const getCreateConversationMock = (): ConversationCreateProps => ({ + title: 'test', + apiConfig: { + connectorId: '1', + connectorTypeTitle: 'test-connector', + defaultSystemPromptId: 'default-system-prompt', + model: 'test-model', + provider: 'OpenAI', + }, + excludeFromLastConversationStorage: false, + isDefault: false, + messages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + replacements: {} as any, +}); describe('createConversation', () => { beforeEach(() => { @@ -18,62 +34,84 @@ describe('createConversation', () => { jest.clearAllMocks(); }); - test('it returns a list as expected with the id changed out for the elastic id', async () => { - const options = getCreateListOptionsMock(); + test('it returns a conversation as expected with the id changed out for the elastic id', async () => { + const options = getCreateConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const list = await createConversation({ ...options, esClient }); - const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; - expect(list).toEqual(expected); + const conversation = await createConversation( + esClient, + 'index-1', + 'test', + { name: 'test' }, + options + ); + const expected: CreateMessageSchema = { + ...getconversationResponseMock(), + id: 'elastic-id-123', + }; + expect(conversation).toEqual(expected); }); - test('it returns a list as expected with the id changed out for the elastic id and seralizer and deseralizer set', async () => { - const options: CreateListOptions = { - ...getCreateListOptionsMock(), - deserializer: '{{value}}', - serializer: '(?)', + test('it returns a conversation as expected with the id changed out for the elastic id and seralizer and deseralizer set', async () => { + const options: CreateconversationOptions = { + ...getCreateConversationMock(), + title: '{{value}}', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const list = await createConversation({ ...options, esClient }); - const expected: ListSchema = { - ...getListResponseMock(), - deserializer: '{{value}}', + const conversation = await createConversation( + esClient, + 'index-1', + 'test', + { name: 'test' }, + options + ); + const expected: CreateMessageSchema = { + ...getconversationResponseMock(), + title: '{{value}}', id: 'elastic-id-123', - serializer: '(?)', }; - expect(list).toEqual(expected); + expect(conversation).toEqual(expected); }); test('It calls "esClient" with body, id, and conversationIndex', async () => { - const options = getCreateListOptionsMock(); - await createConversation(options); - const body = getIndexESListMock(); + const options = getCreateConversationMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + await createConversation(esClient, 'index-1', 'test', { name: 'test' }, options); + const body = getIndexESconversationMock(); const expected = { body, - id: LIST_ID, - index: LIST_INDEX, + id: '1', + index: 'index-1', refresh: 'wait_for', }; expect(options.esClient.create).toBeCalledWith(expected); }); test('It returns an auto-generated id if id is sent in undefined', async () => { - const options = getCreateListOptionsMock(); - options.id = undefined; + const options = getCreateConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const list = await createConversation({ ...options, esClient }); - const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; - expect(list).toEqual(expected); + const conversation = await createConversation( + esClient, + 'index-1', + 'test', + { name: 'test' }, + options + ); + const expected: CreateMessageSchema = { + ...getconversationResponseMock(), + id: 'elastic-id-123', + }; + expect(conversation).toEqual(expected); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index b84ecf4b90dd8..dc5c74a8fc5fb 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -6,11 +6,47 @@ */ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { updateConversation } from './update_conversation'; +import { UpdateConversationSchema, updateConversation } from './update_conversation'; +import { getConversation } from './get_conversation'; +import { + ConversationResponse, + ConversationUpdateProps, +} from '../schemas/conversations/common_attributes.gen'; + +export const getUpdateConversationOptionsMock = (): ConversationUpdateProps => ({ + id: 'test', + title: 'test', + apiConfig: { + connectorId: '1', + connectorTypeTitle: 'test-connector', + defaultSystemPromptId: 'default-system-prompt', + model: 'test-model', + provider: 'OpenAI', + }, + excludeFromLastConversationStorage: false, + messages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + replacements: {} as any, +}); + +export const getConversationResponseMock = (): ConversationResponse => ({ + id: 'test', + title: 'test', + apiConfig: { + connectorId: '1', + connectorTypeTitle: 'test-connector', + defaultSystemPromptId: 'default-system-prompt', + model: 'test-model', + provider: 'OpenAI', + }, + excludeFromLastConversationStorage: false, + messages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + replacements: {} as any, +}); jest.mock('./get_conversation', () => ({ - getList: jest.fn(), + getConversation: jest.fn(), })); describe('updateConversation', () => { @@ -22,45 +58,49 @@ describe('updateConversation', () => { jest.clearAllMocks(); }); - test('it returns a list with serializer and deserializer', async () => { - const list: ListSchema = { - ...getListResponseMock(), - deserializer: '{{value}}', - serializer: '(?)', + test('it returns a conversation with serializer and deserializer', async () => { + const conversation: UpdateConversationSchema = { + ...getConversationResponseMock(), + title: 'test', + '@timestamp': Date.now().toLocaleString(), }; - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getupdateConversationOptionsMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); + const options = getUpdateConversationOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 1 }); - const updatedList = await updateConversation({ ...options, esClient }); - const expected: ListSchema = { - ...getListResponseMock(), - deserializer: '{{value}}', - id: list.id, - serializer: '(?)', + const updatedList = await updateConversation( + esClient, + jest.fn(), + 'index-1', + options, + conversation + ); + const expected: UpdateConversationSchema = { + ...getConversationResponseMock(), + id: conversation.id, + title: 'test', }; expect(updatedList).toEqual(expected); }); - test('it returns null when there is not a list to update', async () => { - (getList as unknown as jest.Mock).mockResolvedValueOnce(null); - const options = getupdateConversationOptionsMock(); + test('it returns null when there is not a conversation to update', async () => { + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getUpdateConversationOptionsMock(); const updatedList = await updateConversation(options); expect(updatedList).toEqual(null); }); - test('throw error if no list was updated', async () => { - const list: ListSchema = { - ...getListResponseMock(), - deserializer: '{{value}}', - serializer: '(?)', + test('throw error if no conversation was updated', async () => { + const conversation: UpdateConversationSchema = { + ...getConversationResponseMock(), + title: 'test', }; - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getupdateConversationOptionsMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); + const options = getUpdateConversationOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 0 }); await expect(updateConversation({ ...options, esClient })).rejects.toThrow( - 'No list has been updated' + 'No conversation has been updated' ); }); }); From 84d684181a986e57bb5211f657e1569fb1545afe Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 25 Jan 2024 05:49:18 +0000 Subject: [PATCH 046/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/elastic_assistant/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 89394105d24b7..f9983a4d5a91b 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -40,7 +40,6 @@ "@kbn/core-elasticsearch-client-server-mocks", "@kbn/task-manager-plugin", "@kbn/security-plugin", - "@kbn/securitysolution-io-ts-list-types", "@kbn/es-query", "@kbn/es-types", "@kbn/config-schema", From 2ca468173dc088503f285977392a10c1de98cb27 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 25 Jan 2024 13:49:56 -0800 Subject: [PATCH 047/141] fixed lint issues --- .../select_system_prompt/index.tsx | 3 +- .../create_conversation.test.ts | 117 ++++++++++------- .../create_conversation.ts | 33 ++--- .../delete_conversation.test.ts | 119 ++++++++---------- .../delete_conversation.ts | 15 ++- .../get_conversation.test.ts | 57 ++++++++- .../server/conversations_data_client/index.ts | 28 ++--- .../update_conversation.test.ts | 68 ++++++---- .../update_conversation.ts | 25 ++-- 9 files changed, 278 insertions(+), 187 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx index fa01249a2b64d..cca352838d826 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/select_system_prompt/index.tsx @@ -73,8 +73,7 @@ const SelectSystemPromptComponent: React.FC = ({ (prompt: Prompt | undefined) => { if (conversation) { setApiConfig({ - conversationId: conversation.id, - title: conversation.title, + conversation, apiConfig: { ...conversation.apiConfig, defaultSystemPromptId: prompt?.id, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index db5277e144512..6b7a2da52a460 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -6,8 +6,11 @@ */ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import { CreateMessageSchema, createConversation } from './create_conversation'; -import { ConversationCreateProps } from '../schemas/conversations/common_attributes.gen'; +import { createConversation } from './create_conversation'; +import { + ConversationCreateProps, + ConversationResponse, +} from '../schemas/conversations/common_attributes.gen'; export const getCreateConversationMock = (): ConversationCreateProps => ({ title: 'test', @@ -25,6 +28,27 @@ export const getCreateConversationMock = (): ConversationCreateProps => ({ replacements: {} as any, }); +export const getConversationResponseMock = (): ConversationResponse => ({ + id: 'test', + title: 'test', + apiConfig: { + connectorId: '1', + connectorTypeTitle: 'test-connector', + defaultSystemPromptId: 'default-system-prompt', + model: 'test-model', + provider: 'OpenAI', + }, + excludeFromLastConversationStorage: false, + messages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + replacements: {} as any, + createdAt: Date.now().toLocaleString(), + namespace: 'default', + isDefault: false, + updatedAt: Date.now().toLocaleString(), + timestamp: Date.now().toLocaleString(), +}); + describe('createConversation', () => { beforeEach(() => { jest.clearAllMocks(); @@ -35,28 +59,31 @@ describe('createConversation', () => { }); test('it returns a conversation as expected with the id changed out for the elastic id', async () => { - const options = getCreateConversationMock(); + const conversation = getCreateConversationMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const conversation = await createConversation( + const createdConversation = await createConversation({ esClient, - 'index-1', - 'test', - { name: 'test' }, - options - ); - const expected: CreateMessageSchema = { - ...getconversationResponseMock(), + conversationIndex: 'index-1', + spaceId: 'test', + user: { name: 'test' }, + conversation, + }); + + const expected: ConversationResponse = { + ...getConversationResponseMock(), id: 'elastic-id-123', }; - expect(conversation).toEqual(expected); + + expect(createdConversation).toEqual(expected); }); - test('it returns a conversation as expected with the id changed out for the elastic id and seralizer and deseralizer set', async () => { - const options: CreateconversationOptions = { + test('it returns a conversation as expected with the id changed out for the elastic id and title set', async () => { + const conversation: ConversationCreateProps = { ...getCreateConversationMock(), title: '{{value}}', }; @@ -65,53 +92,55 @@ describe('createConversation', () => { // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const conversation = await createConversation( + const createdConversation = await createConversation({ esClient, - 'index-1', - 'test', - { name: 'test' }, - options - ); - const expected: CreateMessageSchema = { - ...getconversationResponseMock(), - title: '{{value}}', + conversationIndex: 'index-1', + spaceId: 'test', + user: { name: 'test' }, + conversation, + }); + + const expected: ConversationResponse = { + ...getConversationResponseMock(), id: 'elastic-id-123', + title: 'test new title', }; - expect(conversation).toEqual(expected); + expect(createdConversation).toEqual(expected); }); test('It calls "esClient" with body, id, and conversationIndex', async () => { - const options = getCreateConversationMock(); + const conversation = getCreateConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - await createConversation(esClient, 'index-1', 'test', { name: 'test' }, options); - const body = getIndexESconversationMock(); - const expected = { - body, - id: '1', - index: 'index-1', - refresh: 'wait_for', - }; - expect(options.esClient.create).toBeCalledWith(expected); + await createConversation({ + esClient, + conversationIndex: 'index-1', + spaceId: 'test', + user: { name: 'test' }, + conversation, + }); + + expect(esClient.create).toBeCalled(); }); test('It returns an auto-generated id if id is sent in undefined', async () => { - const options = getCreateConversationMock(); + const conversation = getCreateConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); - const conversation = await createConversation( + const createdConversation = await createConversation({ esClient, - 'index-1', - 'test', - { name: 'test' }, - options - ); - const expected: CreateMessageSchema = { - ...getconversationResponseMock(), + conversationIndex: 'index-1', + spaceId: 'test', + user: { name: 'test' }, + conversation, + }); + + const expected: ConversationResponse = { + ...getConversationResponseMock(), id: 'elastic-id-123', }; - expect(conversation).toEqual(expected); + expect(createdConversation).toEqual(expected); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index e60b6cfd30032..4d3da0eb0520b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -54,20 +54,23 @@ export interface CreateMessageSchema { namespace: string; } -export const createConversation = async ( - esClient: ElasticsearchClient, - conversationIndex: string, - namespace: string, - user: { id?: UUID; name?: string }, - conversation: ConversationCreateProps -): Promise => { +export interface CreateConversationParams { + esClient: ElasticsearchClient; + conversationIndex: string; + spaceId: string; + user: { id?: UUID; name?: string }; + conversation: ConversationCreateProps; +} + +export const createConversation = async ({ + esClient, + conversationIndex, + spaceId, + user, + conversation, +}: CreateConversationParams): Promise => { const createdAt = new Date().toISOString(); - const body: CreateMessageSchema = transformToCreateScheme( - createdAt, - namespace, - user, - conversation - ); + const body: CreateMessageSchema = transformToCreateScheme(createdAt, spaceId, user, conversation); const response = await esClient.index({ body, @@ -85,7 +88,7 @@ export const createConversation = async ( export const transformToCreateScheme = ( createdAt: string, - namespace: string, + spaceId: string, user: { id?: UUID; name?: string }, { title, @@ -125,7 +128,7 @@ export const transformToCreateScheme = ( })), updated_at: createdAt, replacements, - namespace, + namespace: spaceId, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index dd2a8639e9278..d01e7134142a8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -5,22 +5,43 @@ * 2.0. */ -import { getListResponseMock } from '../../../common/schemas/response/list_schema.mock'; -import { LIST_ID, LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { DeleteConversationParams, deleteConversation } from './delete_conversation'; +import { getConversation } from './get_conversation'; +import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; -import { getList } from './get_list'; -import { deleteList } from './delete_list'; -import { getDeleteListOptionsMock } from './delete_list.mock'; - -jest.mock('../utils', () => ({ - waitUntilDocumentIndexed: jest.fn(), +jest.mock('./get_conversation', () => ({ + getConversation: jest.fn(), })); -jest.mock('./get_list', () => ({ - getList: jest.fn(), -})); +export const getConversationResponseMock = (): ConversationResponse => ({ + id: 'test', + title: 'test', + apiConfig: { + connectorId: '1', + connectorTypeTitle: 'test-connector', + defaultSystemPromptId: 'default-system-prompt', + model: 'test-model', + provider: 'OpenAI', + }, + excludeFromLastConversationStorage: false, + messages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + replacements: {} as any, + createdAt: Date.now().toLocaleString(), + namespace: 'default', + isDefault: false, + updatedAt: Date.now().toLocaleString(), + timestamp: Date.now().toLocaleString(), +}); -describe('delete_list', () => { +export const getDeleteConversationOptionsMock = (): DeleteConversationParams => ({ + esClient: elasticsearchClientMock.createScopedClusterClient().asCurrentUser, + id: 'test', + conversationIndex: '', +}); + +describe('deleteConversation', () => { beforeEach(() => { jest.clearAllMocks(); }); @@ -29,71 +50,35 @@ describe('delete_list', () => { jest.clearAllMocks(); }); - test('Delete returns a null if the list is also null', async () => { - (getList as unknown as jest.Mock).mockResolvedValueOnce(null); - const options = getDeleteListOptionsMock(); - const deletedList = await deleteList(options); + test('Delete returns a null if the conversation is also null', async () => { + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteConversationOptionsMock(); + const deletedList = await deleteConversation(options); expect(deletedList).toEqual(null); }); - test('Delete returns the list if a list is returned from getList', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getDeleteListOptionsMock(); - options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); - const deletedList = await deleteList(options); - expect(deletedList).toEqual(list); - }); - - test('Delete calls "deleteByQuery" for list items if a list is returned from getList', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getDeleteListOptionsMock(); - options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); - await deleteList(options); - const deleteByQuery = { - body: { query: { term: { list_id: LIST_ID } } }, - conflicts: 'proceed', - index: LIST_ITEM_INDEX, - refresh: false, - }; - expect(options.esClient.deleteByQuery).toHaveBeenNthCalledWith(1, deleteByQuery); - }); - - test('Delete calls "deleteByQuery" for list if a list is returned from getList', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getDeleteListOptionsMock(); + test('Delete returns the conversation if a conversation is returned from getConversation', async () => { + const conversation = getConversationResponseMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); + const options = getDeleteConversationOptionsMock(); options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); - await deleteList(options); - const deleteByQuery = { - body: { - query: { - ids: { - values: [LIST_ID], - }, - }, - }, - conflicts: 'proceed', - index: LIST_INDEX, - refresh: false, - }; - expect(options.esClient.deleteByQuery).toHaveBeenCalledWith(deleteByQuery); + const deletedList = await deleteConversation(options); + expect(deletedList).toEqual(conversation); }); - test('Delete does not call data client if the list returns null', async () => { - (getList as unknown as jest.Mock).mockResolvedValueOnce(null); - const options = getDeleteListOptionsMock(); - await deleteList(options); + test('Delete does not call data client if the conversation returns null', async () => { + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteConversationOptionsMock(); + await deleteConversation(options); expect(options.esClient.delete).not.toHaveBeenCalled(); }); - test('throw error if no list was deleted', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getDeleteListOptionsMock(); + test('throw error if no conversation was deleted', async () => { + const conversation = getConversationResponseMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); + const options = getDeleteConversationOptionsMock(); options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 0 }); - await expect(deleteList(options)).rejects.toThrow('No list has been deleted'); + await expect(deleteConversation(options)).rejects.toThrow('No conversation has been deleted'); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index eb66807841821..c8eeec2e00e7b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -8,11 +8,16 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { getConversation } from './get_conversation'; -export const deleteConversation = async ( - esClient: ElasticsearchClient, - conversationIndex: string, - id: string -): Promise => { +export interface DeleteConversationParams { + esClient: ElasticsearchClient; + conversationIndex: string; + id: string; +} +export const deleteConversation = async ({ + esClient, + conversationIndex, + id, +}: DeleteConversationParams): Promise => { const conversation = await getConversation(esClient, conversationIndex, id); if (conversation !== null) { const response = await esClient.deleteByQuery({ diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index d5f422abd7ded..7bc45138d3c3d 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -7,6 +7,51 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { getConversation } from './get_conversation'; +import { estypes } from '@elastic/elasticsearch'; +import { SearchEsConversationSchema } from './types'; +import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; + +export const getConversationResponseMock = (): ConversationResponse => ({ + createdAt: '2020-04-20T15:25:31.830Z', + title: 'title-1', + updatedAt: '2020-04-20T15:25:31.830Z', + messages: [], + id: '1', + namespace: 'default', +}); + +export const getSearchConversationMock = + (): estypes.SearchResponse => ({ + _scroll_id: '123', + _shards: { + failed: 0, + skipped: 0, + successful: 0, + total: 0, + }, + hits: { + hits: [ + { + _id: '1', + _index: '', + _score: 0, + _source: { + '@timestamp': '2020-04-20T15:25:31.830Z', + created_at: '2020-04-20T15:25:31.830Z', + title: 'title-1', + updated_at: '2020-04-20T15:25:31.830Z', + messages: [], + id: '1', + namespace: 'default', + }, + }, + ], + max_score: 0, + total: 1, + }, + timed_out: false, + took: 10, + }); describe('getConversation', () => { beforeEach(() => { @@ -17,21 +62,21 @@ describe('getConversation', () => { jest.clearAllMocks(); }); - test('it returns a list as expected if the list is found', async () => { - const data = getSearchListMock(); + test('it returns a conversation as expected if the conversation is found', async () => { + const data = getSearchConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation(esClient, LIST_INDEX, id); - const expected = getListResponseMock(); + const conversation = await getConversation(esClient, '', '1'); + const expected = getConversationResponseMock(); expect(conversation).toEqual(expected); }); test('it returns null if the search is empty', async () => { - const data = getSearchListMock(); + const data = getSearchConversationMock(); data.hits.hits = []; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation(esClient, LIST_INDEX, id); + const conversation = await getConversation(esClient, '', '1'); expect(conversation).toEqual(null); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index a8698053beedd..1a2620ae2bc84 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -181,17 +181,17 @@ export class AIAssistantConversationsDataClient { * @returns The conversation created */ public createConversation = async ( - props: ConversationCreateProps + conversation: ConversationCreateProps ): Promise => { const { currentUser } = this; const esClient = await this.options.elasticsearchClientPromise; - return createConversation( + return createConversation({ esClient, - this.indexTemplateAndPattern.alias, - this.spaceId, - { id: currentUser?.profile_uid, name: currentUser?.username }, - props - ); + conversationIndex: this.indexTemplateAndPattern.alias, + spaceId: this.spaceId, + user: { id: currentUser?.profile_uid, name: currentUser?.username }, + conversation, + }); }; /** @@ -211,14 +211,14 @@ export class AIAssistantConversationsDataClient { isPatch?: boolean ): Promise => { const esClient = await this.options.elasticsearchClientPromise; - return updateConversation( + return updateConversation({ esClient, - this.options.logger, - this.indexTemplateAndPattern.alias, + logger: this.options.logger, + conversationIndex: this.indexTemplateAndPattern.alias, existingConversation, - updatedProps, - isPatch - ); + conversation: updatedProps, + isPatch, + }); }; /** @@ -229,6 +229,6 @@ export class AIAssistantConversationsDataClient { */ public deleteConversation = async (id: string): Promise => { const esClient = await this.options.elasticsearchClientPromise; - deleteConversation(esClient, this.indexTemplateAndPattern.alias, id); + deleteConversation({ esClient, conversationIndex: this.indexTemplateAndPattern.alias, id }); }; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index dc5c74a8fc5fb..59429af9af453 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -6,7 +6,8 @@ */ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import { UpdateConversationSchema, updateConversation } from './update_conversation'; +import { loggerMock } from '@kbn/logging-mocks'; +import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; import { ConversationResponse, @@ -43,6 +44,11 @@ export const getConversationResponseMock = (): ConversationResponse => ({ messages: [], // eslint-disable-next-line @typescript-eslint/no-explicit-any replacements: {} as any, + createdAt: Date.now().toLocaleString(), + namespace: 'default', + isDefault: false, + updatedAt: Date.now().toLocaleString(), + timestamp: Date.now().toLocaleString(), }); jest.mock('./get_conversation', () => ({ @@ -59,23 +65,21 @@ describe('updateConversation', () => { }); test('it returns a conversation with serializer and deserializer', async () => { - const conversation: UpdateConversationSchema = { - ...getConversationResponseMock(), - title: 'test', - '@timestamp': Date.now().toLocaleString(), - }; - (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); - const options = getUpdateConversationOptionsMock(); + const conversation: ConversationUpdateProps = getUpdateConversationOptionsMock(); + const existingConversation = getConversationResponseMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(existingConversation); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 1 }); - const updatedList = await updateConversation( + + const updatedList = await updateConversation({ esClient, - jest.fn(), - 'index-1', - options, - conversation - ); - const expected: UpdateConversationSchema = { + logger: loggerMock.create(), + conversationIndex: 'index-1', + existingConversation, + conversation, + }); + const expected: ConversationResponse = { ...getConversationResponseMock(), id: conversation.id, title: 'test', @@ -85,22 +89,34 @@ describe('updateConversation', () => { test('it returns null when there is not a conversation to update', async () => { (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); - const options = getUpdateConversationOptionsMock(); - const updatedList = await updateConversation(options); + const conversation = getUpdateConversationOptionsMock(); + const existingConversation = getConversationResponseMock(); + + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + const updatedList = await updateConversation({ + esClient, + logger: loggerMock.create(), + conversationIndex: 'index-1', + existingConversation, + conversation, + }); expect(updatedList).toEqual(null); }); test('throw error if no conversation was updated', async () => { - const conversation: UpdateConversationSchema = { - ...getConversationResponseMock(), - title: 'test', - }; - (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); - const options = getUpdateConversationOptionsMock(); + const existingConversation = getConversationResponseMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(existingConversation); + const conversation = getUpdateConversationOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 0 }); - await expect(updateConversation({ ...options, esClient })).rejects.toThrow( - 'No conversation has been updated' - ); + await expect( + updateConversation({ + esClient, + logger: loggerMock.create(), + conversationIndex: 'index-1', + existingConversation, + conversation, + }) + ).rejects.toThrow('No conversation has been updated'); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 47a2aa5184db6..4a863d1a4f701 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -43,14 +43,23 @@ export interface UpdateConversationSchema { updated_at?: string; } -export const updateConversation = async ( - esClient: ElasticsearchClient, - logger: Logger, - conversationIndex: string, - existingConversation: ConversationResponse, - conversation: ConversationUpdateProps, - isPatch?: boolean -): Promise => { +export interface UpdateConversationParams { + esClient: ElasticsearchClient; + logger: Logger; + conversationIndex: string; + existingConversation: ConversationResponse; + conversation: ConversationUpdateProps; + isPatch?: boolean; +} + +export const updateConversation = async ({ + esClient, + logger, + conversationIndex, + existingConversation, + conversation, + isPatch, +}: UpdateConversationParams): Promise => { const updatedAt = new Date().toISOString(); const params = transformToUpdateScheme(updatedAt, conversation); From 90ec31b32c7044b37599dc181c1bff8ca235a81a Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 25 Jan 2024 16:31:07 -0800 Subject: [PATCH 048/141] - --- .../connector_setup/index.test.tsx | 3 +- .../__mocks__/conversations_schema.mock.ts | 103 +++++++++++------- .../server/__mocks__/response.ts | 3 +- ...reate_resource_installation_helper.test.ts | 4 - .../server/lib/langchain/types.ts | 4 - .../server/lib/model_evaluator/evaluation.ts | 4 +- .../find_user_conversations_route.test.ts | 3 - .../routes/conversations/read_route.test.ts | 3 - .../knowledge_base/get_kb_resource.test.ts | 25 ++++- .../evaluate/post_evaluate_route.gen.ts | 2 +- .../evaluate/post_evaluate_route.schema.yaml | 1 + 11 files changed, 90 insertions(+), 65 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx index 1bf6b6bc6ff2b..4abb8998cf18f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.test.tsx @@ -35,7 +35,7 @@ jest.mock('../add_connector_modal', () => ({ ), })); -const setApiConfig = jest.fn(); +const setApiConfig = jest.fn().mockResolvedValue(welcomeConvo); const mockConversation = { appendMessage: jest.fn(), appendReplacements: jest.fn(), @@ -138,7 +138,6 @@ describe('useConnectorSetup', () => { expect(clearTimeout).toHaveBeenCalled(); expect(onSetupComplete).toHaveBeenCalled(); - expect(onConversationUpdate).toHaveBeenCalled(); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 9944a9efe7b86..76ff0725f8bc3 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -5,56 +5,75 @@ * 2.0. */ -export const getCreateConversationSchemaMock = (ruleId = 'rule-1'): QueryRuleCreateProps => ({ - description: 'Detecting root and admin users', - name: 'Query with a rule id', - query: 'user.name: root or user.name: admin', - severity: 'high', - type: 'query', - risk_score: 55, - language: 'kuery', - rule_id: ruleId, +import { PerformBulkActionRequestBody } from '../schemas/conversations/bulk_crud_conversations_route.gen'; +import { + ConversationCreateProps, + ConversationResponse, + ConversationUpdateProps, +} from '../schemas/conversations/common_attributes.gen'; + +export const getCreateConversationSchemaMock = (): ConversationCreateProps => ({ + title: 'Welcome', + apiConfig: { + connectorId: '1', + defaultSystemPromptId: 'Default', + connectorTypeTitle: 'Test connector', + model: 'model', + }, + excludeFromLastConversationStorage: false, + isDefault: true, + messages: [ + { + content: 'test content', + role: 'user', + timestamp: '2019-12-13T16:40:33.400Z', + traceData: { + traceId: '1', + transactionId: '2', + }, + }, + ], }); -export const getUpdateConversationSchemaMock = (ruleId = 'rule-1'): QueryRuleCreateProps => ({ - description: 'Detecting root and admin users', - name: 'Query with a rule id', - query: 'user.name: root or user.name: admin', - severity: 'high', - type: 'query', - risk_score: 55, - language: 'kuery', - rule_id: ruleId, +export const getUpdateConversationSchemaMock = ( + conversationId = 'conversation-1' +): ConversationUpdateProps => ({ + title: 'Welcome 2', + apiConfig: { + connectorId: '2', + defaultSystemPromptId: 'Default', + connectorTypeTitle: 'Test connector', + model: 'model', + }, + excludeFromLastConversationStorage: false, + messages: [ + { + content: 'test content', + role: 'user', + timestamp: '2019-12-13T16:40:33.400Z', + traceData: { + traceId: '1', + transactionId: '2', + }, + }, + ], + id: conversationId, }); -export const getConversationMock = (params: T): SanitizedRule => ({ +export const getConversationMock = ( + params: ConversationCreateProps | ConversationUpdateProps +): ConversationResponse => ({ id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - name: 'Detect Root/Admin Users', - tags: [], - alertTypeId: ruleTypeMappings[params.type], - consumer: 'siem', - params, - createdAt: new Date('2019-12-13T16:40:33.400Z'), - updatedAt: new Date('2019-12-13T16:40:33.400Z'), - schedule: { interval: '5m' }, - enabled: true, - actions: [], - throttle: null, - notifyWhen: null, - createdBy: 'elastic', - updatedBy: 'elastic', - apiKeyOwner: 'elastic', - muteAll: false, - mutedInstanceIds: [], - scheduledTaskId: '2dabe330-0702-11ea-8b50-773b89126888', - executionStatus: { - status: 'unknown', - lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + ...params, + createdAt: '2019-12-13T16:40:33.400Z', + updatedAt: '2019-12-13T16:40:33.400Z', + namespace: 'default', + user: { + name: 'elastic', }, - revision: 0, }); -export const getQueryConversationParams = (): QueryRuleParams => { +export const getQueryConversationParams = (): ConversationCreateProps | ConversationUpdateProps => { return { ...getBaseRuleParams(), type: 'query', diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts index 16f3e652e5b0b..870572d57b7fb 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts @@ -8,12 +8,13 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { getConversationMock, getQueryConversationParams } from './conversations_schema.mock'; import { estypes } from '@elastic/elasticsearch'; +import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; export const responseMock = { create: httpServerMock.createResponseFactory, }; -export interface FindHit { +export interface FindHit { page: number; perPage: number; total: number; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts index 49a8bed94bca7..38547af124ad4 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts @@ -151,10 +151,6 @@ describe('createResourceInstallationHelper', () => { }); test(`should retry using new common init function if specified`, async () => { - const context = { - context: 'test1', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - }; const helper = createResourceInstallationHelper( logger, getCommonInitPromise(false, 100), diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts index 8529a870a0243..1fe5074a0b733 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/types.ts @@ -5,10 +5,6 @@ * 2.0. */ -import { PostActionsConnectorExecuteBodyInputs } from '../../schemas/actions_connector/post_actions_connector_execute'; - -export type RequestBody = PostActionsConnectorExecuteBodyInputs; - export interface ResponseBody { data: string; connector_id: string; diff --git a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts index 54040d3d1b58e..71ed301ba71fe 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts @@ -13,10 +13,10 @@ import { Logger } from '@kbn/core/server'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer, RunCollectorCallbackHandler } from 'langchain/callbacks'; import { AgentExecutorEvaluatorWithMetadata } from '../langchain/executors/types'; -import { Dataset } from '../../schemas/evaluate/post_evaluate'; import { callAgentWithRetry, getMessageFromLangChainResponse } from './utils'; -import { ResponseBody } from '../langchain/types'; import { isLangSmithEnabled, writeLangSmithFeedback } from '../../routes/evaluate/utils'; +import { Dataset } from '../../schemas/evaluate/post_evaluate_route.gen'; +import { ResponseBody } from '../langchain/types'; export interface PerformEvaluationParams { agentExecutorEvaluators: AgentExecutorEvaluatorWithMetadata[]; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts index 5b270e5ad9996..b24c439f20ae4 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { loggingSystemMock } from '@kbn/core/server/mocks'; import { getFindRequest, requestMock } from '../../__mocks__/request'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS } from '@kbn/elastic-assistant-common'; import { serverMock } from '../../__mocks__/server'; @@ -20,11 +19,9 @@ import { describe('Find user conversations route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); - let logger: ReturnType; beforeEach(async () => { server = serverMock.create(); - logger = loggingSystemMock.createLogger(); ({ clients, context } = requestContextMock.createTools()); clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts index 2fc645ad52c1b..7d1027c5d82fb 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { loggingSystemMock } from '@kbn/core/server/mocks'; import { requestContextMock } from '../../__mocks__/request_context'; import { serverMock } from '../../__mocks__/server'; import { readConversationRoute } from './read_route'; @@ -20,12 +19,10 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assis describe('Read conversation route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); - let logger: ReturnType; const myFakeId = '99403909-ca9b-49ba-9d7a-7e5320e68d05'; beforeEach(() => { server = serverMock.create(); - logger = loggingSystemMock.createLogger(); ({ clients, context } = requestContextMock.createTools()); clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.test.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.test.ts index 7c4a9058e7df7..292f68555ad23 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { KibanaRequest } from '@kbn/core/server'; import { getKbResource } from './get_kb_resource'; describe('getKbResource', () => { @@ -15,7 +16,13 @@ describe('getKbResource', () => { }); it('returns undefined when params is undefined', () => { - const request = { params: undefined }; + const request = { params: undefined } as unknown as KibanaRequest< + { resource?: string | undefined }, + unknown, + unknown, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any + >; const result = getKbResource(request); @@ -23,7 +30,13 @@ describe('getKbResource', () => { }); it('returns undefined when resource is undefined', () => { - const request = { params: { resource: undefined } }; + const request = { params: { resource: undefined } } as unknown as KibanaRequest< + { resource?: string | undefined }, + unknown, + unknown, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any + >; const result = getKbResource(request); @@ -31,7 +44,13 @@ describe('getKbResource', () => { }); it('returns the decoded resource', () => { - const request = { params: { resource: 'esql%20query' } }; + const request = { params: { resource: 'esql%20query' } } as unknown as KibanaRequest< + { resource?: string | undefined }, + unknown, + unknown, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any + >; const result = getKbResource(request); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts index 737aa132ead74..306922f7242f3 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts @@ -19,7 +19,7 @@ import { z } from 'zod'; export type DatasetItem = z.infer; export const DatasetItem = z.object({ id: z.string(), - input: z.string().optional(), + input: z.string(), reference: z.string(), tags: z.array(z.string()).optional(), prediction: z.string().optional(), diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml index 4d7e189216094..209c6d748cb27 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml @@ -89,6 +89,7 @@ components: required: - id - reference + - input properties: id: type: string From df0c262f978aa4b51ea6972ff1e4e62f9d1a6817 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 25 Jan 2024 20:11:58 -0800 Subject: [PATCH 049/141] fixed security_solution tests --- .../__mocks__/conversations_schema.mock.ts | 73 ++++++++++++------ .../server/ai_assistant_service/index.test.ts | 74 ++----------------- .../server/ai_assistant_service/index.ts | 2 +- .../routes/conversations/create_route.test.ts | 41 ++-------- .../use_assistant_telemetry/index.test.tsx | 29 ++++---- .../use_conversation_store/index.test.tsx | 54 +++++++++++++- 6 files changed, 131 insertions(+), 142 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 76ff0725f8bc3..1ff0cf5da97d7 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -73,31 +73,60 @@ export const getConversationMock = ( }, }); -export const getQueryConversationParams = (): ConversationCreateProps | ConversationUpdateProps => { - return { - ...getBaseRuleParams(), - type: 'query', - language: 'kuery', - query: 'user.name: root or user.name: admin', - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - dataViewId: undefined, - filters: [ - { - query: { - match_phrase: { - 'host.name': 'some-host', +export const getQueryConversationParams = ( + isUpdate?: boolean +): ConversationCreateProps | ConversationUpdateProps => { + return isUpdate + ? { + title: 'Welcome 2', + apiConfig: { + connectorId: '2', + defaultSystemPromptId: 'Default', + connectorTypeTitle: 'Test connector', + model: 'model', + }, + excludeFromLastConversationStorage: false, + messages: [ + { + content: 'test content', + role: 'user', + timestamp: '2019-12-13T16:40:33.400Z', + traceData: { + traceId: '1', + transactionId: '2', + }, }, + ], + id: '1', + } + : { + title: 'Welcome', + apiConfig: { + connectorId: '1', + defaultSystemPromptId: 'Default', + connectorTypeTitle: 'Test connector', + model: 'model', }, - }, - ], - savedId: undefined, - alertSuppression: undefined, - responseActions: undefined, - }; + excludeFromLastConversationStorage: false, + isDefault: true, + messages: [ + { + content: 'test content', + role: 'user', + timestamp: '2019-12-13T16:40:33.400Z', + traceData: { + traceId: '1', + transactionId: '2', + }, + }, + ], + }; }; export const getPerformBulkActionSchemaMock = (): PerformBulkActionRequestBody => ({ - query: '', - ids: undefined, - action: BulkActionTypeEnum.disable, + create: [], + delete: { + ids: [], + }, + update: [], }); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index b31b70ccb6950..1d5ce98265bca 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -10,11 +10,9 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { ReplaySubject, Subject } from 'rxjs'; -import { IRuleTypeAlerts, RecoveredActionGroup } from '../types'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; import { retryUntil } from './test_utils'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; -import { UntypedNormalizedRuleType } from '../rule_type_registry'; -import { getDataStreamAdapter } from './lib/data_stream_adapter'; import { conversationsDataClientMock } from '../__mocks__/conversations_data_client.mock'; import { AIAssistantConversationsDataClient } from '../conversations_data_client'; import { AIAssistantService } from '.'; @@ -107,16 +105,14 @@ interface GetIndexTemplatePutBodyOpts { useDataStream?: boolean; } const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { - const context = opts ? opts.context : undefined; const namespace = (opts ? opts.namespace : undefined) ?? DEFAULT_NAMESPACE_STRING; - const useLegacyAlerts = opts ? opts.useLegacyAlerts : undefined; const useEcs = opts ? opts.useEcs : undefined; const secondaryAlias = opts ? opts.secondaryAlias : undefined; const useDataStream = opts?.useDataStream ?? false; const indexPatterns = useDataStream - ? [`.alerts-${context ? context : 'test'}.alerts-${namespace}`] - : [`.internal.alerts-${context ? context : 'test'}.alerts-${namespace}-*`]; + ? [`.alerts-.alerts-${namespace}`] + : [`.internal.alerts-.alerts-${namespace}-*`]; return { name: `.alerts-${context ? context : 'test'}.alerts-${namespace}-index-template`, body: { @@ -171,15 +167,8 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { }; }; -const TestRegistrationContext: IRuleTypeAlerts = { - context: 'test', - mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, - shouldWrite: true, -}; - const getContextInitialized = async ( assistantService: AIAssistantService, - context: string = TestRegistrationContext.context, namespace: string = DEFAULT_NAMESPACE_STRING ) => { const { result } = await assistantService.getSpaceResourcesInitializationPromise(namespace); @@ -187,30 +176,6 @@ const getContextInitialized = async ( }; const conversationsDataClient = conversationsDataClientMock.create(); -const ruleType: jest.Mocked = { - id: 'test.rule-type', - name: 'My test rule', - actionGroups: [{ id: 'default', name: 'Default' }, RecoveredActionGroup], - defaultActionGroupId: 'default', - minimumLicenseRequired: 'basic', - isExportable: true, - recoveryActionGroup: RecoveredActionGroup, - executor: jest.fn(), - category: 'test', - producer: 'alerts', - cancelAlertsOnRuleTimeout: true, - ruleTaskTimeout: '5m', - autoRecoverAlerts: true, - validate: { - params: { validate: (params) => params }, - }, - validLegacyConsumers: [], -}; - -const ruleTypeWithAlertDefinition: jest.Mocked = { - ...ruleType, - alerts: TestRegistrationContext as IRuleTypeAlerts<{}>, -}; describe('AI Assistant Service', () => { let pluginStop$: Subject; @@ -245,7 +210,7 @@ describe('AI Assistant Service', () => { elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, kibanaVersion: '8.8.0', - dataStreamAdapter, + taskManager: taskManagerMock.createSetup(), }); await retryUntil( @@ -270,29 +235,6 @@ describe('AI Assistant Service', () => { expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); }); - test('should log error and set initialized to false if adding ILM policy throws error', async () => { - if (useDataStreamForAlerts) return; - - clusterClient.ilm.putLifecycle.mockRejectedValueOnce(new Error('fail')); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing ILM policy .alerts-ilm-policy - fail` - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(1); - }); - test('should log error and set initialized to false if creating/updating common component template throws error', async () => { clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); const assistantService = new AIAssistantService({ @@ -300,7 +242,7 @@ describe('AI Assistant Service', () => { elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, kibanaVersion: '8.8.0', - dataStreamAdapter, + taskManager: taskManagerMock.createSetup(), }); await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); @@ -380,11 +322,11 @@ describe('AI Assistant Service', () => { elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, kibanaVersion: '8.8.0', - dataStreamAdapter, + taskManager: taskManagerMock.createSetup(), }); await retryUntil( - 'alert service initialized', + 'assistant service initialized', async () => assistantService.isInitialized() === true ); @@ -421,7 +363,7 @@ describe('AI Assistant Service', () => { elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, kibanaVersion: '8.8.0', - dataStreamAdapter, + taskManager: taskManagerMock.createSetup(), }); await retryUntil( diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index d2ca3d4c28f2a..80487472b1e39 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -69,7 +69,7 @@ export class AIAssistantService { ); } - public async isInitialized() { + public isInitialized() { return this.initialized; } diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts index 784cd48e6058e..718eb1ca81669 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts @@ -116,39 +116,10 @@ describe('Create conversation route', () => { expect(result.badRequest).toHaveBeenCalled(); }); - - test('allows rule type of query and custom from and interval', async () => { - const request = requestMock.create({ - method: 'post', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, - body: { from: 'now-7m', interval: '5m', ...getCreateConversationSchemaMock() }, - }); - const result = server.validate(request); - - expect(result.ok).toHaveBeenCalled(); - }); - - test('disallows invalid "from" param on rule', async () => { - const request = requestMock.create({ - method: 'post', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, - body: { - from: 'now-3755555555555555.67s', - interval: '5m', - ...getCreateConversationSchemaMock(), - }, - }); - const result = server.validate(request); - expect(result.badRequest).toHaveBeenCalledWith('from: Failed to parse date-math expression'); - }); }); - describe('rule containing response actions', () => { - beforeEach(() => { - // @ts-expect-error We're writting to a read only property just for the purpose of the test - clients.config.experimentalFeatures.endpointResponseActionsEnabled = true; - }); + describe('conversation containing messages', () => { const getResponseAction = (command: string = 'isolate') => ({ - action_type_id: '.endpoint', + role: 'user', params: { command, comment: '', @@ -170,20 +141,20 @@ describe('Create conversation route', () => { expect(response.status).toEqual(200); }); - test('fails when provided with an unsupported command', async () => { - const wrongAction = getResponseAction('processes'); + test('fails when provided with an unsupported message role', async () => { + const wrongMessage = getResponseAction('test_thing'); const request = requestMock.create({ method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, body: { ...getCreateConversationSchemaMock(), - response_actions: [wrongAction], + messages: [wrongMessage], }, }); const result = await server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'response_actions.0.action_type_id: Invalid literal value, expected ".osquery", response_actions.0.params.command: Invalid literal value, expected "isolate"' + 'messages.0.role: Invalid literal value, expected "user", messages.0.params.command: Invalid literal value, expected "isolate"' ); }); }); diff --git a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.test.tsx index a61a740394c7b..1ecc3e47ab063 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_assistant_telemetry/index.test.tsx @@ -30,14 +30,6 @@ const mockedTelemetry = { reportAssistantSettingToggled: () => {}, }; -jest.mock('../use_conversation_store', () => { - return { - useConversationStore: () => ({ - conversations: mockedConversations, - }), - }; -}); - jest.mock('../../common/lib/kibana', () => { const original = jest.requireActual('../../common/lib/kibana'); @@ -51,6 +43,15 @@ jest.mock('../../common/lib/kibana', () => { }; }); +jest.mock('@kbn/elastic-assistant', () => ({ + getConversationById: jest.fn().mockReturnValue({ + id: customId, + title: 'Custom', + apiConfig: {}, + messages: [], + }), +})); + const trackingFns = [ 'reportAssistantInvoked', 'reportAssistantMessageSent', @@ -69,12 +70,12 @@ describe('useAssistantTelemetry', () => { }); describe.each(trackingFns)('Handles %s id masking', (fn) => { - it('Should call tracking with appropriate id when tracking is called with an isDefault=true conversation id', () => { + it('Should call tracking with appropriate id when tracking is called with an isDefault=true conversation id', async () => { const { result } = renderHook(() => useAssistantTelemetry()); const validId = Object.keys(mockedConversations)[0]; // @ts-ignore const trackingFn = result.current[fn]; - trackingFn({ conversationId: validId, invokedBy: 'shortcut' }); + await trackingFn({ conversationId: validId, invokedBy: 'shortcut' }); // @ts-ignore const trackingMockedFn = mockedTelemetry[fn]; expect(trackingMockedFn).toHaveBeenCalledWith({ @@ -83,11 +84,11 @@ describe('useAssistantTelemetry', () => { }); }); - it('Should call tracking with "Custom" id when tracking is called with an isDefault=false conversation id', () => { + it('Should call tracking with "Custom" id when tracking is called with an isDefault=false conversation id', async () => { const { result } = renderHook(() => useAssistantTelemetry()); // @ts-ignore const trackingFn = result.current[fn]; - trackingFn({ conversationId: customId, invokedBy: 'shortcut' }); + await trackingFn({ conversationId: customId, invokedBy: 'shortcut' }); // @ts-ignore const trackingMockedFn = mockedTelemetry[fn]; expect(trackingMockedFn).toHaveBeenCalledWith({ @@ -96,11 +97,11 @@ describe('useAssistantTelemetry', () => { }); }); - it('Should call tracking with "Custom" id when tracking is called with an unknown conversation id', () => { + it('Should call tracking with "Custom" id when tracking is called with an unknown conversation id', async () => { const { result } = renderHook(() => useAssistantTelemetry()); // @ts-ignore const trackingFn = result.current[fn]; - trackingFn({ conversationId: '123', invokedBy: 'shortcut' }); + await trackingFn({ conversationId: '123', invokedBy: 'shortcut' }); // @ts-ignore const trackingMockedFn = mockedTelemetry[fn]; expect(trackingMockedFn).toHaveBeenCalledWith({ diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx index 6a9c890dcb874..e71cedf8f955c 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx @@ -12,6 +12,7 @@ import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-da import { useKibana } from '../../common/lib/kibana'; import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { unset } from 'lodash/fp'; +import { useFetchCurrentUserConversations } from '@kbn/elastic-assistant'; const BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY = unset( DATA_QUALITY_DASHBOARD_CONVERSATION_ID, @@ -22,6 +23,14 @@ jest.mock('../../common/links', () => ({ useLinkAuthorized: jest.fn(), })); +jest.mock('@kbn/elastic-assistant', () => ({ + useFetchCurrentUserConversations: jest.fn().mockReturnValue({ + data: {}, + isLoading: false, + isError: false, + }), +})); + const mockedUseKibana = { ...mockUseKibana(), services: { @@ -51,17 +60,54 @@ describe('useConversationStore', () => { (useLinkAuthorized as jest.Mock).mockReturnValue(true); const { result } = renderHook(() => useConversationStore()); - expect(result.current.conversations).toEqual( - expect.objectContaining(BASE_SECURITY_CONVERSATIONS) - ); + expect(result.current).toEqual(expect.objectContaining(BASE_SECURITY_CONVERSATIONS)); }); it('should return conversations Without "Data Quality dashboard" conversation', () => { (useLinkAuthorized as jest.Mock).mockReturnValue(false); const { result } = renderHook(() => useConversationStore()); - expect(result.current.conversations).toEqual( + expect(result.current).toEqual( + expect.objectContaining(BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY) + ); + }); + + it('should return stored conversations merged with the base conversations', () => { + (useLinkAuthorized as jest.Mock).mockReturnValue(true); + + const persistedConversations = { + data: { + '1234': { + id: '1234', + title: 'Welcome', + isDefault: true, + messages: [], + apiConfig: { + connectorId: 'c29c28a0-20fe-11ee-9306-a1f4d42ec542', + provider: 'OpenAi', + }, + }, + '5657': { + id: '5657', + title: 'Test', + isDefault: true, + messages: [], + apiConfig: { + connectorId: 'c29c28a0-20fe-11ee-9306-a1f4d42ec542', + provider: 'OpenAi', + }, + }, + }, + isLoading: false, + isError: false, + }; + (useFetchCurrentUserConversations as jest.Mock).mockReturnValue(persistedConversations); + const { result } = renderHook(() => useConversationStore()); + + expect(result.current).toEqual( expect.objectContaining(BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY) ); + + expect(result.current).toEqual(expect.objectContaining(persistedConversations.data)); }); }); From d893192bd07da7f8843835b972ce4a72b7a705cb Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 25 Jan 2024 21:36:11 -0800 Subject: [PATCH 050/141] useConversation tests --- .../assistant/use_conversation/index.test.tsx | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index 8d4d1abb31406..c0b581cf7bd22 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -11,34 +11,17 @@ import { TestProviders } from '../../mock/test_providers/test_providers'; import { welcomeConvo } from '../../mock/conversation'; import React from 'react'; import { ConversationRole } from '../../assistant_context/types'; +import { httpServiceMock } from '@kbn/core/public/mocks'; +import { WELCOME_CONVERSATION } from './sample_conversations'; import { - appendConversationMessagesApi, + appendConversationMessagesApi as _appendConversationMessagesApi, deleteConversationApi, - getConversationById, + getConversationById as _getConversationById, updateConversationApi, + createConversationApi as _createConversationApi, } from '../api/conversations'; -import { httpServiceMock } from '@kbn/core/public/mocks'; -import { WELCOME_CONVERSATION } from './sample_conversations'; - -jest.mock('../api/conversations', () => { - const actual = jest.requireActual('../api/conversations'); - return { - ...actual, - updateConversationApi: jest.fn((...args) => actual.updateConversationApi(...args)), - appendConversationMessagesApi: jest.fn((...args) => - actual.appendConversationMessagesApi(...args) - ), - createConversationApi: jest.fn((...args) => actual.createConversationApi(...args)), - deleteConversationApi: jest.fn((...args) => actual.deleteConversationApi(...args)), - getConversationById: jest.fn((...args) => actual.getConversationById(...args)), - }; -}); - -const updateConversationApiMock = updateConversationApi as jest.Mock; -const getConversationByIdMock = getConversationById as jest.Mock; -const deleteConversationApiMock = deleteConversationApi as jest.Mock; -const appendConversationMessagesApiMock = appendConversationMessagesApi as jest.Mock; +jest.mock('../api/conversations'); const message = { content: 'You are a robot', role: 'user' as ConversationRole, @@ -57,6 +40,10 @@ const mockConvo = { apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, }; +const appendConversationMessagesApi = _appendConversationMessagesApi as jest.Mock; +const getConversationById = _getConversationById as jest.Mock; +const createConversationApi = _createConversationApi as jest.Mock; + describe('useConversation', () => { let httpMock: ReturnType; @@ -71,6 +58,9 @@ describe('useConversation', () => { wrapper: ({ children }) => {children}, }); await waitForNextUpdate(); + appendConversationMessagesApi.mockResolvedValue({ + messages: [message, anotherMessage, message], + }); const appendResult = await result.current.appendMessage({ conversationId: welcomeConvo.id, @@ -101,6 +91,8 @@ describe('useConversation', () => { ), }); await waitForNextUpdate(); + + appendConversationMessagesApi.mockResolvedValue([message, anotherMessage, message]); await result.current.appendMessage({ conversationId: welcomeConvo.id, message, @@ -117,9 +109,12 @@ describe('useConversation', () => { it('should create a new conversation when called with valid conversationId and message', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); + createConversationApi.mockResolvedValue(mockConvo); const createResult = await result.current.createConversation({ id: mockConvo.id, @@ -135,13 +130,15 @@ describe('useConversation', () => { it('should delete an existing conversation when called with valid conversationId', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); await result.current.deleteConversation('new-convo'); - expect(deleteConversationApiMock).toHaveBeenCalledWith({ + expect(deleteConversationApi).toHaveBeenCalledWith({ http: httpMock, id: 'new-convo', }); @@ -151,7 +148,9 @@ describe('useConversation', () => { it('should update the apiConfig for an existing conversation when called with a valid conversationId and apiConfig', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); @@ -160,10 +159,9 @@ describe('useConversation', () => { apiConfig: mockConvo.apiConfig, }); - expect(updateConversationApiMock).toHaveBeenCalledWith({ + expect(createConversationApi).toHaveBeenCalledWith({ http: httpMock, - conversationId: welcomeConvo.id, - apiConfig: mockConvo.apiConfig, + conversation: { ...WELCOME_CONVERSATION, apiConfig: mockConvo.apiConfig, id: '' }, }); }); }); @@ -171,11 +169,13 @@ describe('useConversation', () => { it('appends replacements', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); - getConversationByIdMock.mockResolvedValue(mockConvo); + getConversationById.mockResolvedValue(mockConvo); await result.current.appendReplacements({ conversationId: welcomeConvo.id, @@ -186,7 +186,7 @@ describe('useConversation', () => { }, }); - expect(updateConversationApiMock).toHaveBeenCalledWith({ + expect(updateConversationApi).toHaveBeenCalledWith({ http: httpMock, conversationId: welcomeConvo.id, replacements: { @@ -201,11 +201,13 @@ describe('useConversation', () => { it('should remove the last message from a conversation when called with valid conversationId', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); - getConversationByIdMock.mockResolvedValue(mockConvo); + getConversationById.mockResolvedValue(mockConvo); const removeResult = await result.current.removeLastMessage('new-convo'); @@ -216,7 +218,9 @@ describe('useConversation', () => { it('amendMessage updates the last message of conversation[] for a given conversationId with provided content', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, + wrapper: ({ children }) => ( + {children} + ), }); await waitForNextUpdate(); @@ -225,7 +229,7 @@ describe('useConversation', () => { content: 'hello world', }); - expect(appendConversationMessagesApiMock).toHaveBeenCalledWith({ + expect(appendConversationMessagesApi).toHaveBeenCalledWith({ conversationId: mockConvo.id, http: httpMock, messages: [ From 74d75b86b71f9211da6c0e8d7627e23986c9fb4e Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 25 Jan 2024 21:51:46 -0800 Subject: [PATCH 051/141] fixed types checks --- .../server/lib/executor.test.ts | 4 +- .../elastic_assistant/server/lib/executor.ts | 4 +- .../server/lib/langchain/executors/types.ts | 5 +- .../lib/langchain/llm/actions_client_llm.ts | 6 +-- .../server/routes/evaluate/post_evaluate.ts | 4 +- .../server/routes/helpers.test.ts | 8 ++-- ...ost_actions_connector_execute_route.gen.ts | 46 +++++++++---------- ...ctions_connector_execute_route.schema.yaml | 2 + .../plugins/elastic_assistant/server/types.ts | 4 +- .../alert_counts/alert_counts_tool.test.ts | 8 ++-- .../esql_language_knowledge_base_tool.test.ts | 4 +- .../open_and_acknowledged_alerts_tool.test.ts | 8 ++-- 12 files changed, 52 insertions(+), 51 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts index def4eca415d42..11aab9d753335 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts @@ -14,13 +14,13 @@ import { executeAction, Props } from './executor'; import { PassThrough } from 'stream'; import { KibanaRequest } from '@kbn/core-http-server'; -import { RequestBody } from './langchain/types'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; +import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; const request = { body: { params: {}, }, -} as KibanaRequest; +} as KibanaRequest; describe('executeAction', () => { beforeEach(() => { diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index 27064f3fb1961..9e832ee90edff 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -9,12 +9,12 @@ import { get } from 'lodash/fp'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { KibanaRequest } from '@kbn/core-http-server'; import { PassThrough, Readable } from 'stream'; -import { RequestBody } from './langchain/types'; +import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; export interface Props { actions: ActionsPluginStart; connectorId: string; - request: KibanaRequest; + request: KibanaRequest; } interface StaticResponse { connector_id: string; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts index 03554911216eb..3fa3b207fcc23 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts @@ -12,7 +12,8 @@ import { Logger } from '@kbn/logging'; import { KibanaRequest } from '@kbn/core-http-server'; import type { LangChainTracer } from 'langchain/callbacks'; import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server'; -import { RequestBody, ResponseBody } from '../types'; +import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ResponseBody } from '../types'; import type { AssistantTool } from '../../../types'; export interface AgentExecutorParams { @@ -30,7 +31,7 @@ export interface AgentExecutorParams { logger: Logger; onNewReplacements?: (newReplacements: Record) => void; replacements?: Record; - request: KibanaRequest; + request: KibanaRequest; size?: number; elserId?: string; traceOptions?: TraceOptions; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts index ca1afdeef7368..9e504c2c98221 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts @@ -11,8 +11,8 @@ import type { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plu import { LLM } from 'langchain/llms/base'; import { get } from 'lodash/fp'; +import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; import { getMessageContentAndRole } from '../helpers'; -import { RequestBody } from '../types'; const LLM_TYPE = 'ActionsClientLlm'; @@ -21,7 +21,7 @@ interface ActionsClientLlmParams { connectorId: string; llmType?: string; logger: Logger; - request: KibanaRequest; + request: KibanaRequest; traceId?: string; } @@ -29,7 +29,7 @@ export class ActionsClientLlm extends LLM { #actions: ActionsPluginStart; #connectorId: string; #logger: Logger; - #request: KibanaRequest; + #request: KibanaRequest; #actionResultData: string; #traceId: string; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index b2cf67291704d..c400c21ab4d44 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -27,13 +27,13 @@ import { setupEvaluationIndex, } from '../../lib/model_evaluator/output_index/utils'; import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils'; -import { RequestBody } from '../../lib/langchain/types'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; import { EvaluateRequestBody, EvaluateRequestQuery, } from '../../schemas/evaluate/post_evaluate_route.gen'; import { buildRouteValidationWithZod } from '../route_validation'; +import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; /** * To support additional Agent Executors from the UI, add them to this map @@ -143,7 +143,7 @@ export const postEvaluateRoute = ( // Skeleton request from route to pass to the agents // params will be passed to the actions executor - const skeletonRequest: KibanaRequest = { + const skeletonRequest: KibanaRequest = { ...request, body: { alertsIndexPattern: '', diff --git a/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts b/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts index 384e1f8865736..f298b4a17e0a2 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts @@ -6,9 +6,9 @@ */ import type { KibanaRequest } from '@kbn/core-http-server'; -import type { RequestBody } from '../lib/langchain/types'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from './helpers'; +import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('getPluginNameFromRequest', () => { const contextRequestHeaderEncoded = encodeURIComponent( @@ -25,7 +25,7 @@ describe('getPluginNameFromRequest', () => { headers: { 'x-kbn-context': contextRequestHeaderEncoded, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; beforeEach(() => { jest.clearAllMocks(); @@ -44,7 +44,7 @@ describe('getPluginNameFromRequest', () => { headers: { 'x-kbn-context': undefined, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const pluginName = getPluginNameFromRequest({ request: invalidRequest, defaultPluginName: DEFAULT_PLUGIN_NAME, @@ -57,7 +57,7 @@ describe('getPluginNameFromRequest', () => { headers: { 'x-kbn-context': 'asdfku', }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const pluginName = getPluginNameFromRequest({ request: invalidRequest, defaultPluginName: DEFAULT_PLUGIN_NAME, diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 85b530c0c153f..4f28d619e9560 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -27,30 +27,28 @@ export type ExecuteConnectorRequestParamsInput = z.input; export const ExecuteConnectorRequestBody = z.object({ - params: z - .object({ - subActionParams: z - .object({ - messages: z - .array( - z.object({ - /** - * Message role. - */ - role: z.enum(['system', 'user', 'assistant']).optional(), - content: z.string().optional(), - }) - ) - .optional(), - model: z.string().optional(), - n: z.number().optional(), - stop: z.array(z.string()).optional(), - temperature: z.number().optional(), - }) - .optional(), - subAction: z.string().optional(), - }) - .optional(), + params: z.object({ + subActionParams: z + .object({ + messages: z + .array( + z.object({ + /** + * Message role. + */ + role: z.enum(['system', 'user', 'assistant']).optional(), + content: z.string().optional(), + }) + ) + .optional(), + model: z.string().optional(), + n: z.number().optional(), + stop: z.array(z.string()).optional(), + temperature: z.number().optional(), + }) + .optional(), + subAction: z.string().optional(), + }), alertsIndexPattern: z.string().optional(), allow: z.array(z.string()).optional(), allowReplacement: z.array(z.string()).optional(), diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index 2e5dd417c4024..f3d1a7d5064da 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -24,6 +24,8 @@ paths: application/json: schema: type: object + required: + - params properties: params: type: object diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index d82e89af95255..d13effa9a1f75 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -29,9 +29,9 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { AssistantFeatures } from '@kbn/elastic-assistant-common'; import { AIAssistantConversationsDataClient } from './conversations_data_client'; import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; -import { RequestBody } from './lib/langchain/types'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; import { AIAssistantAnonimizationFieldsSOClient } from './saved_object/ai_assistant_anonimization_fields_so_client'; +import { ExecuteConnectorRequestBody } from './schemas/actions_connector/post_actions_connector_execute_route.gen'; export const PLUGIN_ID = 'elasticAssistant' as const; @@ -196,6 +196,6 @@ export interface AssistantToolParams { modelExists: boolean; onNewReplacements?: (newReplacements: Record) => void; replacements?: Record; - request: KibanaRequest; + request: KibanaRequest; size?: number; } diff --git a/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts index 79b9c2e171f0a..7139192869106 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts @@ -10,9 +10,9 @@ import type { KibanaRequest } from '@kbn/core-http-server'; import type { DynamicTool } from 'langchain/tools'; import { omit } from 'lodash/fp'; -import type { RequestBody } from '@kbn/elastic-assistant-plugin/server/lib/langchain/types'; import { ALERT_COUNTS_TOOL } from './alert_counts_tool'; import type { RetrievalQAChain } from 'langchain/chains'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('AlertCountsTool', () => { const alertsIndexPattern = 'alerts-index'; @@ -29,7 +29,7 @@ describe('AlertCountsTool', () => { replacements, size: 20, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const isEnabledKnowledgeBase = true; const chain = {} as unknown as RetrievalQAChain; const modelExists = true; @@ -61,7 +61,7 @@ describe('AlertCountsTool', () => { alertsIndexPattern: '.alerts-security.alerts-default', size: 20, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const params = { esClient, request: requestMissingAnonymizationParams, @@ -165,7 +165,7 @@ describe('AlertCountsTool', () => { const requestWithMissingParams = omit('body.allow', request) as unknown as KibanaRequest< unknown, unknown, - RequestBody + ExecuteConnectorRequestBody >; const tool = ALERT_COUNTS_TOOL.getTool({ diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts index c9605c902f0ca..82dff5bd8b52a 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts @@ -10,7 +10,7 @@ import type { DynamicTool } from 'langchain/tools'; import { ESQL_KNOWLEDGE_BASE_TOOL } from './esql_language_knowledge_base_tool'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { KibanaRequest } from '@kbn/core-http-server'; -import type { RequestBody } from '@kbn/elastic-assistant-plugin/server/lib/langchain/types'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('EsqlLanguageKnowledgeBaseTool', () => { const chain = {} as RetrievalQAChain; @@ -26,7 +26,7 @@ describe('EsqlLanguageKnowledgeBaseTool', () => { replacements: { key: 'value' }, size: 20, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const rest = { chain, esClient, diff --git a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts index 019fc0075679f..b90bd2fb0c096 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts @@ -11,10 +11,10 @@ import type { DynamicTool } from 'langchain/tools'; import { omit } from 'lodash/fp'; import { OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL } from './open_and_acknowledged_alerts_tool'; -import type { RequestBody } from '@kbn/elastic-assistant-plugin/server/lib/langchain/types'; import { MAX_SIZE } from './helpers'; import type { RetrievalQAChain } from 'langchain/chains'; import { mockAlertsFieldsApi } from '@kbn/elastic-assistant-plugin/server/__mocks__/alerts'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('OpenAndAcknowledgedAlertsTool', () => { const alertsIndexPattern = 'alerts-index'; @@ -31,7 +31,7 @@ describe('OpenAndAcknowledgedAlertsTool', () => { replacements, size: 20, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const isEnabledKnowledgeBase = true; const chain = {} as unknown as RetrievalQAChain; const modelExists = true; @@ -64,7 +64,7 @@ describe('OpenAndAcknowledgedAlertsTool', () => { alertsIndexPattern: '.alerts-security.alerts-default', size: 20, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const params = { request: requestMissingAnonymizationParams, ...rest, @@ -218,7 +218,7 @@ describe('OpenAndAcknowledgedAlertsTool', () => { const requestWithMissingParams = omit('body.allow', request) as unknown as KibanaRequest< unknown, unknown, - RequestBody + ExecuteConnectorRequestBody >; const tool = OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL.getTool({ From bd60b77ea6507efe6352cab7ec09e2a560a084bf Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 28 Jan 2024 20:55:42 -0800 Subject: [PATCH 052/141] api tests fix --- .../group3/type_registrations.test.ts | 2 + .../__mocks__/conversations_schema.mock.ts | 22 +- .../server/__mocks__/request.ts | 51 +- .../server/__mocks__/request_context.ts | 14 +- .../server/__mocks__/response.ts | 2 +- ...reate_resource_installation_helper.test.ts | 26 +- .../server/ai_assistant_service/index.test.ts | 2586 ++++------------- .../conversations_data_writer.test.ts | 139 +- .../conversations_data_writer.ts | 56 +- .../create_conversation.test.ts | 23 +- .../create_conversation.ts | 3 +- .../delete_conversation.test.ts | 10 +- .../delete_conversation.ts | 4 +- .../get_conversation.test.ts | 41 +- .../conversations_data_client/index.test.ts | 219 +- .../server/conversations_data_client/index.ts | 14 +- .../update_conversation.test.ts | 6 +- .../update_conversation.ts | 4 + ...append_conversation_messages_route.test.ts | 133 + .../conversations/bulk_actions_route.test.ts | 246 +- .../conversations/bulk_actions_route.ts | 20 +- .../routes/conversations/create_route.test.ts | 36 +- .../routes/conversations/create_route.ts | 15 + .../routes/conversations/delete_route.test.ts | 29 +- .../routes/conversations/find_route.test.ts | 27 +- .../find_user_conversations_route.test.ts | 40 +- .../routes/conversations/read_route.test.ts | 26 +- .../server/routes/conversations/read_route.ts | 13 +- .../routes/conversations/update_route.test.ts | 66 +- .../post_actions_connector_execute.test.ts | 284 +- 30 files changed, 1617 insertions(+), 2540 deletions(-) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index 99e2692523f6d..527b0eff9ca16 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -44,6 +44,8 @@ const previouslyRegisteredTypes = [ 'csp-rule-template', 'csp_rule', 'dashboard', + 'elastic-ai-assistant-anonimization-fields', + 'elastic-ai-assistant-prompts', 'event-annotation-group', 'endpoint:user-artifact', 'endpoint:user-artifact-manifest', diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 1ff0cf5da97d7..8556fb34f9e5d 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -11,6 +11,7 @@ import { ConversationResponse, ConversationUpdateProps, } from '../schemas/conversations/common_attributes.gen'; +import { AppendConversationMessageRequestBody } from '../schemas/conversations/crud_conversation_route.gen'; export const getCreateConversationSchemaMock = (): ConversationCreateProps => ({ title: 'Welcome', @@ -60,6 +61,21 @@ export const getUpdateConversationSchemaMock = ( id: conversationId, }); +export const getAppendConversationMessagesSchemaMock = + (): AppendConversationMessageRequestBody => ({ + messages: [ + { + content: 'test content', + role: 'user', + timestamp: '2019-12-13T16:40:33.400Z', + traceData: { + traceId: '1', + transactionId: '2', + }, + }, + ], + }); + export const getConversationMock = ( params: ConversationCreateProps | ConversationUpdateProps ): ConversationResponse => ({ @@ -124,9 +140,9 @@ export const getQueryConversationParams = ( }; export const getPerformBulkActionSchemaMock = (): PerformBulkActionRequestBody => ({ - create: [], + create: [getQueryConversationParams(false) as ConversationCreateProps], delete: { - ids: [], + ids: ['99403909-ca9b-49ba-9d7a-7e5320e68d05'], }, - update: [], + update: [getQueryConversationParams(true) as ConversationUpdateProps], }); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 9d6f2b977d605..cd58896bbb45b 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -14,12 +14,19 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, } from '@kbn/elastic-assistant-common'; import { + getAppendConversationMessagesSchemaMock, getCreateConversationSchemaMock, getUpdateConversationSchemaMock, } from './conversations_schema.mock'; +import { + ConversationCreateProps, + ConversationUpdateProps, +} from '../schemas/conversations/common_attributes.gen'; export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -66,17 +73,23 @@ export const getPostEvaluateRequest = ({ query, }); -export const getFindRequest = () => +export const getCurrentUserFindRequest = () => requestMock.create({ method: 'get', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, }); -export const getDeleteConversationRequest = () => +export const getFindRequest = () => + requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, + }); + +export const getDeleteConversationRequest = (id: string = '04128c15-0d1b-4716-a4c5-46997ac7f3bd') => requestMock.create({ method: 'delete', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - query: { id: 'conversation-1' }, + params: { id }, }); export const getCreateConversationRequest = () => @@ -86,36 +99,44 @@ export const getCreateConversationRequest = () => body: getCreateConversationSchemaMock(), }); -export const getUpdateConversationRequest = () => +export const getUpdateConversationRequest = (id: string = '04128c15-0d1b-4716-a4c5-46997ac7f3bd') => requestMock.create({ method: 'put', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, body: getUpdateConversationSchemaMock(), + params: { id }, }); -export const getConversationReadRequest = () => +export const getAppendConversationMessageRequest = ( + id: string = '04128c15-0d1b-4716-a4c5-46997ac7f3bd' +) => requestMock.create({ - method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - query: { id: 'conversation-1' }, + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, + body: getAppendConversationMessagesSchemaMock(), + params: { id }, }); -export const getConversationReadRequestWithId = (id: string) => +export const getConversationReadRequest = (id: string = '04128c15-0d1b-4716-a4c5-46997ac7f3bd') => requestMock.create({ method: 'get', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - query: { id }, + params: { id }, }); -export const getConversationsBulkActionRequest = () => +export const getConversationsBulkActionRequest = ( + create: ConversationCreateProps[] = [], + update: ConversationUpdateProps[] = [], + deleteIds: string[] = [] +) => requestMock.create({ method: 'patch', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, body: { - create: [], - update: [], + create, + update, delete: { - ids: [], + ids: deleteIds, }, }, }); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index 2dcf83d34cc4c..f508fa047bd1e 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -15,6 +15,7 @@ import { } from '../types'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { conversationsDataClientMock } from './conversations_data_client.mock'; +import { AIAssistantConversationsDataClient } from '../conversations_data_client'; export const createMockClients = () => { const core = coreMock.createRequestHandlerContext(); @@ -32,6 +33,8 @@ export const createMockClients = () => { getAIAssistantConversationsDataClient: conversationsDataClientMock.create(), getAIAssistantPromptsSOClient: jest.fn(), getAIAssistantAnonimizationFieldsSOClient: jest.fn(), + getSpaceId: jest.fn(), + getCurrentUser: jest.fn(), }, savedObjectsClient: core.savedObjects.client, @@ -82,7 +85,16 @@ const createElasticAssistantRequestContextMock = ( getRegisteredFeatures: jest.fn(), getRegisteredTools: jest.fn(), logger: clients.elasticAssistant.logger, - getAIAssistantConversationsDataClient: jest.fn(), + + getAIAssistantConversationsDataClient: jest.fn( + () => clients.elasticAssistant.getAIAssistantConversationsDataClient + ) as unknown as jest.MockInstance< + Promise, + [], + unknown + > & + (() => Promise), + getAIAssistantPromptsSOClient: jest.fn(), getAIAssistantAnonimizationFieldsSOClient: jest.fn(), getCurrentUser: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts index 870572d57b7fb..b18485feaf01b 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts @@ -32,7 +32,7 @@ export const getFindConversationsResultWithSingleHit = (): FindHit => ({ page: 1, perPage: 1, total: 1, - data: [getConversationMock(getQueryConversationParams())], + data: [getConversationMock(getQueryConversationParams(true))], }); export const getBasicEmptySearchResponse = (): estypes.SearchResponse => ({ diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts index 38547af124ad4..915ceb1032b2b 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts @@ -88,19 +88,19 @@ describe('createResourceInstallationHelper', () => { } ); - // Add two contexts that need to be initialized + // Add two namespaces that need to be initialized helper.add('test1'); helper.add('test2'); await retryUntil('init fns run', async () => logger.info.mock.calls.length === 3); expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); - expect(logger.info).toHaveBeenNthCalledWith(2, 'test1_default'); - expect(logger.info).toHaveBeenNthCalledWith(3, 'test2_default'); - expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ + // expect(logger.info).toHaveBeenNthCalledWith(2, 'test1_default'); + // expect(logger.info).toHaveBeenNthCalledWith(3, 'test2_default'); + expect(await helper.getInitializedResources('test1')).toEqual({ result: true, }); - expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources('test2')).toEqual({ result: true, }); }); @@ -119,9 +119,9 @@ describe('createResourceInstallationHelper', () => { await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); expect(logger.warn).toHaveBeenCalledWith( - `Common resources were not initialized, cannot initialize context for test1` + `Common resources were not initialized, cannot initialize resources for test1` ); - expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(await helper.getInitializedResources('test1')).toEqual({ result: false, error: `error initializing`, }); @@ -143,8 +143,8 @@ describe('createResourceInstallationHelper', () => { async () => (await getContextInitialized(helper)) === false ); - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - fail`); - expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ + expect(logger.error).toHaveBeenCalledWith(`Error initializing resources test1 - fail`); + expect(await helper.getInitializedResources('test1')).toEqual({ result: false, error: `fail`, }); @@ -164,7 +164,7 @@ describe('createResourceInstallationHelper', () => { await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); expect(logger.warn).toHaveBeenCalledWith( - `Common resources were not initialized, cannot initialize context for test1` + `Common resources were not initialized, cannot initialize resources for default` ); expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, @@ -206,7 +206,7 @@ describe('createResourceInstallationHelper', () => { async () => (await getContextInitialized(helper)) === false ); - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - first error`); + expect(logger.error).toHaveBeenCalledWith(`Error initializing resources default - first error`); expect(await helper.getInitializedResources(DEFAULT_NAMESPACE_STRING)).toEqual({ result: false, error: `first error`, @@ -223,7 +223,9 @@ describe('createResourceInstallationHelper', () => { return logger.error.mock.calls.length === 1; }); - expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - second error`); + expect(logger.error).toHaveBeenCalledWith( + `Error initializing resources default - second error` + ); // the second retry is throttled so this is never called expect(logger.info).not.toHaveBeenCalledWith('test1_default successfully retried'); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index 1d5ce98265bca..4510d14e5cf1a 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -6,16 +6,16 @@ */ import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { ReplaySubject, Subject } from 'rxjs'; import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; -import { retryUntil } from './test_utils'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import { conversationsDataClientMock } from '../__mocks__/conversations_data_client.mock'; import { AIAssistantConversationsDataClient } from '../conversations_data_client'; import { AIAssistantService } from '.'; +import { retryUntil } from './create_resource_installation_helper.test'; jest.mock('../conversations_data_client'); @@ -36,22 +36,9 @@ const SimulateTemplateResponse = { settings: {}, }, }; -interface HTTPError extends Error { - statusCode: number; -} - -interface EsError extends Error { - meta: { - body: { - error: { - type: string; - }; - }; - }; -} const GetAliasResponse = { - '.internal.alerts-test.alerts-default-000001': { + '.kibana-elastic-ai-assistant-conversations-default-000001': { aliases: { alias_1: { is_hidden: true, @@ -77,97 +64,7 @@ const GetDataStreamResponse: IndicesGetDataStreamResponse = { ], }; -const IlmPutBody = { - policy: { - _meta: { - managed: true, - }, - phases: { - hot: { - actions: { - rollover: { - max_age: '30d', - max_primary_shard_size: '50gb', - }, - }, - }, - }, - }, - name: '.alerts-ilm-policy', -}; - -interface GetIndexTemplatePutBodyOpts { - context?: string; - namespace?: string; - useLegacyAlerts?: boolean; - useEcs?: boolean; - secondaryAlias?: string; - useDataStream?: boolean; -} -const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { - const namespace = (opts ? opts.namespace : undefined) ?? DEFAULT_NAMESPACE_STRING; - const useEcs = opts ? opts.useEcs : undefined; - const secondaryAlias = opts ? opts.secondaryAlias : undefined; - const useDataStream = opts?.useDataStream ?? false; - - const indexPatterns = useDataStream - ? [`.alerts-.alerts-${namespace}`] - : [`.internal.alerts-.alerts-${namespace}-*`]; - return { - name: `.alerts-${context ? context : 'test'}.alerts-${namespace}-index-template`, - body: { - index_patterns: indexPatterns, - composed_of: [ - ...(useEcs ? ['.alerts-ecs-mappings'] : []), - `.alerts-${context ? `${context}.alerts` : 'test.alerts'}-mappings`, - ...(useLegacyAlerts ? ['.alerts-legacy-alert-mappings'] : []), - '.alerts-framework-mappings', - ], - ...(useDataStream ? { data_stream: { hidden: true } } : {}), - priority: namespace.length, - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - ...(useDataStream - ? {} - : { - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`, - }, - }), - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': 2500, - }, - mappings: { - dynamic: false, - _meta: { - kibana: { version: '8.8.0' }, - managed: true, - namespace, - }, - }, - ...(secondaryAlias - ? { - aliases: { - [`${secondaryAlias}-default`]: { - is_write_index: false, - }, - }, - } - : {}), - }, - _meta: { - kibana: { version: '8.8.0' }, - managed: true, - namespace, - }, - }, - }; -}; - -const getContextInitialized = async ( +const getSpaceResourcesInitialized = async ( assistantService: AIAssistantService, namespace: string = DEFAULT_NAMESPACE_STRING ) => { @@ -177,6 +74,14 @@ const getContextInitialized = async ( const conversationsDataClient = conversationsDataClientMock.create(); +const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, +} as AuthenticatedUser; + describe('AI Assistant Service', () => { let pluginStop$: Subject; @@ -198,1926 +103,653 @@ describe('AI Assistant Service', () => { pluginStop$.complete(); }); - for (const useDataStreamForAlerts of [false, true]) { - const label = useDataStreamForAlerts ? 'data streams' : 'aliases'; - const dataStreamAdapter = getDataStreamAdapter({ useDataStreamForAlerts }); - - describe(`using ${label} for alert indices`, () => { - describe('AIAssistantService()', () => { - test('should correctly initialize common resources', async () => { - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - taskManager: taskManagerMock.createSetup(), - }); - - await retryUntil( - 'alert service initialized', - async () => (await assistantService.isInitialized()) === true - ); - - expect(assistantService.isInitialized()).toEqual(true); - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - if (!useDataStreamForAlerts) { - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - } - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); - - const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); - const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; - expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); - const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; - expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); - }); + describe('AIAssistantService()', () => { + test('should correctly initialize common resources', async () => { + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - test('should log error and set initialized to false if creating/updating common component template throws error', async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - taskManager: taskManagerMock.createSetup(), - }); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - expect(logger.error).toHaveBeenCalledWith( - `Error installing component template .alerts-framework-mappings - fail` - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - }); + await retryUntil( + 'AI Assistant service initialized', + async () => (await assistantService.isInitialized()) === true + ); - test('should update index template field limit and retry initialization if creating/updating common component template fails with field limit error', async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( - new EsErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { - error: { - root_cause: [ - { - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - }, - ], - type: 'illegal_argument_exception', - reason: - 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', - caused_by: { - type: 'illegal_argument_exception', - reason: - 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', - caused_by: { - type: 'illegal_argument_exception', - reason: 'Limit of total fields [1900] has been exceeded', - }, - }, - }, - }, - }, - }) - ) - ); - const existingIndexTemplate = { - name: 'test-template', - index_template: { - index_patterns: ['test*'], - composed_of: ['.alerts-framework-mappings'], - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty-default`, - }, - 'index.mapping.total_fields.limit': 1800, - }, - mappings: { - dynamic: false, - }, - }, - }, - }; - clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ - index_templates: [existingIndexTemplate], - }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - taskManager: taskManagerMock.createSetup(), - }); - - await retryUntil( - 'assistant service initialized', - async () => assistantService.isInitialized() === true - ); - - expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(1); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ - name: existingIndexTemplate.name, - body: { - ...existingIndexTemplate.index_template, - template: { - ...existingIndexTemplate.index_template.template, - settings: { - ...existingIndexTemplate.index_template.template?.settings, - 'index.mapping.total_fields.limit': 2500, - }, - }, - }, - }); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - // 3x for framework, legacy-alert and ecs mappings, then 1 extra time to update component template - // after updating index template field limit - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - }); + expect(assistantService.isInitialized()).toEqual(true); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual( + '.kibana-elastic-ai-assistant-component-template-conversations' + ); + }); + + test('should log error and set initialized to false if creating/updating common component template throws error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), }); - describe('register()', () => { - let assistantService: AIAssistantService; - beforeEach(async () => { - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - taskManager: taskManagerMock.createSetup(), - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - }); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - test('should correctly install resources for context when common initialization is complete', async () => { - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - if (!useDataStreamForAlerts) { - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - } else { - expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); - } - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); - const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; - expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); - const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; - expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); - const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; - expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); - - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( - getIndexTemplatePutBody({ useDataStream: useDataStreamForAlerts }) - ); - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalledWith({ - expand_wildcards: 'all', - name: '.alerts-test.alerts-default', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.internal.alerts-test.alerts-default-*', - name: '.alerts-test.alerts-*', - }); - } - }); + expect(assistantService.isInitialized()).toEqual(false); + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template .kibana-elastic-ai-assistant-component-template-conversations - fail` + ); + }); + }); - test('should correctly install resources for custom namespace on demand when isSpaceAware is true', async () => { - assistantService.register({ ...TestRegistrationContext, isSpaceAware: true }); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - if (!useDataStreamForAlerts) { - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - } else { - expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); - } - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( - 1, - getIndexTemplatePutBody({ useDataStream: useDataStreamForAlerts }) - ); - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenNthCalledWith(1, { - expand_wildcards: 'all', - name: '.alerts-test.alerts-default', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenNthCalledWith(1, { - index: '.internal.alerts-test.alerts-default-000001', - body: { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(1, { - index: '.internal.alerts-test.alerts-default-*', - name: '.alerts-test.alerts-*', - }); - } - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - - clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ - data_streams: [], - })); - - await retryUntil( - 'context in namespace initialized', - async () => - (await getContextInitialized( - assistantService, - TestRegistrationContext.context, - 'another-namespace' - )) === true - ); - - expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( - 2, - getIndexTemplatePutBody({ - namespace: 'another-namespace', - useDataStream: useDataStreamForAlerts, - }) - ); - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 4 - ); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 4 - ); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 4 - ); - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).toHaveBeenNthCalledWith(1, { - name: '.alerts-test.alerts-another-namespace', - }); - expect(clusterClient.indices.getDataStream).toHaveBeenNthCalledWith(2, { - expand_wildcards: 'all', - name: '.alerts-test.alerts-another-namespace', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenNthCalledWith(2, { - index: '.internal.alerts-test.alerts-another-namespace-000001', - body: { - aliases: { - '.alerts-test.alerts-another-namespace': { - is_write_index: true, - }, - }, - }, - }); - expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(2, { - index: '.internal.alerts-test.alerts-another-namespace-*', - name: '.alerts-test.alerts-*', - }); - } - }); + describe('createAIAssistantDatastreamClient()', () => { + let assistantService: AIAssistantService; + beforeEach(async () => { + (AIAssistantConversationsDataClient as jest.Mock).mockImplementation( + () => conversationsDataClient + ); + }); - test('should not install component template for context if fieldMap is empty', async () => { - assistantService.register({ - context: 'empty', - mappings: { fieldMap: {} }, - }); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService, 'empty')) === true - ); - - if (!useDataStreamForAlerts) { - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - } else { - expect(clusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); - } - - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); - const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); - const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; - expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); - const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; - expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); - - const template = { - name: `.alerts-empty.alerts-default-index-template`, - body: { - index_patterns: [ - useDataStreamForAlerts - ? `.alerts-empty.alerts-default` - : `.internal.alerts-empty.alerts-default-*`, - ], - composed_of: ['.alerts-framework-mappings'], - ...(useDataStreamForAlerts ? { data_stream: { hidden: true } } : {}), - priority: 7, - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - ...(useDataStreamForAlerts - ? {} - : { - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty.alerts-default`, - }, - }), - 'index.mapping.ignore_malformed': true, - 'index.mapping.total_fields.limit': 2500, - }, - mappings: { - _meta: { - kibana: { version: '8.8.0' }, - managed: true, - namespace: 'default', - }, - dynamic: false, - }, - }, - _meta: { - kibana: { version: '8.8.0' }, - managed: true, - namespace: 'default', - }, - }, - }; - - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith(template); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalledWith({}); - expect(clusterClient.indices.getDataStream).toHaveBeenCalledWith({ - expand_wildcards: 'all', - name: '.alerts-empty.alerts-default', - }); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.internal.alerts-empty.alerts-default-000001', - body: { - aliases: { - '.alerts-empty.alerts-default': { - is_write_index: true, - }, - }, - }, - }); - expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.internal.alerts-empty.alerts-default-*', - name: '.alerts-empty.alerts-*', - }); - } - - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 1 : 2 - ); - }); + test('should create new AIAssistantConversationsDataClient', async () => { + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - test('should skip initialization if context already exists', async () => { - assistantService.register(TestRegistrationContext); - assistantService.register(TestRegistrationContext); + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); - expect(logger.debug).toHaveBeenCalledWith( - `Resources for context "test" have already been registered.` - ); - }); + await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); - test('should throw error if context already exists and has been registered with a different field map', async () => { - assistantService.register(TestRegistrationContext); - expect(() => { - assistantService.register({ - ...TestRegistrationContext, - mappings: { fieldMap: { anotherField: { type: 'keyword', required: false } } }, - }); - }).toThrowErrorMatchingInlineSnapshot( - `"test has already been registered with different options"` - ); - }); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '.kibana-elastic-ai-assistant-conversations', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + }); + }); - test('should throw error if context already exists and has been registered with a different options', async () => { - assistantService.register(TestRegistrationContext); - expect(() => { - assistantService.register({ - ...TestRegistrationContext, - useEcs: true, - }); - }).toThrowErrorMatchingInlineSnapshot( - `"test has already been registered with different options"` - ); - }); + test('should retry initializing common resources if common resource initialization failed', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); - test('should allow same context with different "shouldWrite" option', async () => { - assistantService.register(TestRegistrationContext); - assistantService.register({ - ...TestRegistrationContext, - shouldWrite: false, - }); + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - expect(logger.debug).toHaveBeenCalledWith( - `Resources for context "test" have already been registered.` - ); - }); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - test('should not update index template if simulating template throws error', async () => { - clusterClient.indices.simulateTemplate.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(logger.error).toHaveBeenCalledWith( - `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - fail`, - expect.any(Error) - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - // putIndexTemplate is skipped but other operations are called as expected - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + expect(assistantService.isInitialized()).toEqual(false); - test('should log error and set initialized to false if simulating template returns empty mappings', async () => { - clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ - ...SimulateTemplateResponse, - template: { - ...SimulateTemplateResponse.template, - mappings: {}, - }, - })); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ - result: false, - error: - 'Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping', - }); - - expect(logger.error).toHaveBeenCalledWith( - new Error( - `No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping` - ) - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - }); + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); - test('should log error and set initialized to false if updating index template throws error', async () => { - clusterClient.indices.putIndexTemplate.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ error: 'Failure during installation. fail', result: false }); - - expect(logger.error).toHaveBeenCalledWith( - `Error installing index template .alerts-test.alerts-default-index-template - fail`, - expect.any(Error) - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - }); + const result = await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); - test('should log error and set initialized to false if checking for concrete write index throws error', async () => { - clusterClient.indices.getAlias.mockRejectedValueOnce(new Error('fail')); - clusterClient.indices.getDataStream.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ error: 'Failure during installation. fail', result: false }); - - expect(logger.error).toHaveBeenCalledWith( - useDataStreamForAlerts - ? `Error fetching data stream for .alerts-test.alerts-default - fail` - : `Error fetching concrete indices for .internal.alerts-test.alerts-default-* pattern - fail` - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - }); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - test('should not throw error if checking for concrete write index throws 404', async () => { - const error = new Error(`index doesn't exist`) as HTTPError; - error.statusCode = 404; - clusterClient.indices.getAlias.mockRejectedValueOnce(error); - clusterClient.indices.getDataStream.mockRejectedValueOnce(error); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - } - }); + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - test('should log error and set initialized to false if updating index settings for existing indices throws error', async () => { - clusterClient.indices.putSettings.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ error: 'Failure during installation. fail', result: false }); - - expect(logger.error).toHaveBeenCalledWith( - useDataStreamForAlerts - ? `Failed to PUT index.mapping.total_fields.limit settings for .alerts-test.alerts-default: fail` - : `Failed to PUT index.mapping.total_fields.limit settings for alias_1: fail` - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '.kibana-elastic-ai-assistant-conversations', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + }); - test('should skip updating index mapping for existing indices if simulate index template throws error', async () => { - clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(logger.error).toHaveBeenCalledWith( - useDataStreamForAlerts - ? `Ignored PUT mappings for .alerts-test.alerts-default; error generating simulated mappings: fail` - : `Ignored PUT mappings for alias_1; error generating simulated mappings: fail` - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - - // this is called to update backing indices, so not used with data streams - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - } - }); + expect(result).not.toBe(null); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations`, + 'Retrying common resource initialization', + `Retrying resource initialization for "default"` + ); + }); - test('should log error and set initialized to false if updating index mappings for existing indices throws error', async () => { - clusterClient.indices.putMapping.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ error: 'Failure during installation. fail', result: false }); - - if (useDataStreamForAlerts) { - expect(logger.error).toHaveBeenCalledWith( - `Failed to PUT mapping for .alerts-test.alerts-default: fail` - ); - } else { - expect(logger.error).toHaveBeenCalledWith(`Failed to PUT mapping for alias_1: fail`); - } - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + test('should not retry initializing common resources if common resource initialization is in progress', async () => { + // this is the initial call that fails + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); - test('does not updating settings or mappings if no existing concrete indices', async () => { - clusterClient.indices.getAlias.mockImplementationOnce(async () => ({})); - clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ - data_streams: [], - })); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + // this is the retry call that we'll artificially inflate the duration of + clusterClient.cluster.putComponentTemplate.mockImplementationOnce(async () => { + await new Promise((r) => setTimeout(r, 1000)); + return { acknowledged: true }; + }); - test('should log error and set initialized to false if concrete indices exist but none are write index', async () => { - // not applicable for data streams - if (useDataStreamForAlerts) return; - - clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-0001': { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: false, - is_hidden: true, - }, - alias_2: { - is_write_index: false, - is_hidden: true, - }, - }, - }, - })); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ - error: - 'Failure during installation. Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default', - result: false, - }); - - expect(logger.error).toHaveBeenCalledWith( - new Error( - `Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default` - ) - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - }); + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - test('does not create new index if concrete write index exists', async () => { - // not applicable for data streams - if (useDataStreamForAlerts) return; - - clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-0001': { - aliases: { - '.alerts-test.alerts-default': { - is_write_index: true, - is_hidden: true, - }, - alias_2: { - is_write_index: false, - is_hidden: true, - }, - }, - }, - })); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + // call createAIAssistantConversationsDataClient at the same time which will trigger the retries + const result = await Promise.all([ + assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }), + assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }), + ]); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + + expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); + expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); + + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '.kibana-elastic-ai-assistant-conversations', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + logger, + }); - test('should log error and set initialized to false if create concrete index throws error', async () => { - // not applicable for data streams - if (useDataStreamForAlerts) return; - - clusterClient.indices.create.mockRejectedValueOnce(new Error('fail')); - clusterClient.indices.createDataStream.mockRejectedValueOnce(new Error('fail')); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ error: 'Failure during installation. fail', result: false }); - - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - }); + expect(result[0]).not.toBe(null); + expect(result[1]).not.toBe(null); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.info).toHaveBeenCalledWith( + `Resource installation for "test" succeeded after retry` + ); + expect(logger.info).toHaveBeenCalledWith( + `Skipped retrying common resource initialization because it is already being retried.` + ); + }); - test('should not throw error if create concrete index throws resource_already_exists_exception error and write index already exists', async () => { - // not applicable for data streams - if (useDataStreamForAlerts) return; - - const error = new Error(`fail`) as EsError; - error.meta = { - body: { - error: { - type: 'resource_already_exists_exception', - }, - }, - }; - clusterClient.indices.create.mockRejectedValueOnce(error); - clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-000001': { - aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, - }, - })); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - expect(clusterClient.indices.get).toHaveBeenCalled(); - expect(clusterClient.indices.create).toHaveBeenCalled(); - }); + test('should retry initializing context specific resources if context specific resource initialization failed', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - test('should log error and set initialized to false if create concrete index throws resource_already_exists_exception error and write index does not already exists', async () => { - // not applicable for data streams - if (useDataStreamForAlerts) return; - - const error = new Error(`fail`) as EsError; - error.meta = { - body: { - error: { - type: 'resource_already_exists_exception', - }, - }, - }; - clusterClient.indices.create.mockRejectedValueOnce(error); - clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.internal.alerts-test.alerts-default-000001': { - aliases: { '.alerts-test.alerts-default': { is_write_index: false } }, - }, - })); - - assistantService.register(TestRegistrationContext); - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - expect( - await assistantService.getContextInitializationPromise( - TestRegistrationContext.context, - DEFAULT_NAMESPACE_STRING - ) - ).toEqual({ - error: - 'Failure during installation. Attempted to create index: .internal.alerts-test.alerts-default-000001 as the write index for alias: .alerts-test.alerts-default, but the index already exists and is not the write index for the alias', - result: false, - }); - - expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putMapping).toHaveBeenCalled(); - expect(clusterClient.indices.get).toHaveBeenCalled(); - expect(clusterClient.indices.create).toHaveBeenCalled(); - }); + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + + const result = await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, }); - describe('createAIAssistantDatastreamClient()', () => { - let assistantService: AIAssistantService; - beforeEach(async () => { - (AIAssistantConversationsDataClient as jest.Mock).mockImplementation( - () => conversationsDataClient - ); - }); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '.kibana-elastic-ai-assistant-conversations', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + logger, + }); - test('should create new AlertsClient', async () => { - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - dataStreamAdapter, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - kibanaVersion: '8.8.0', - }); - }); + expect(result).not.toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.info).toHaveBeenCalledWith( + `Resource installation for "test" succeeded after retry` + ); + }); - test('should retry initializing common resources if common resource initialization failed', async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - - // Installing ILM policy failed so no calls to install context-specific resources - // should be made - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - const result = await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 2 - ); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - - expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - dataStreamAdapter, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - kibanaVersion: '8.8.0', - }); - - expect(result).not.toBe(null); - expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.info).toHaveBeenCalledWith( - `Resource installation for "test" succeeded after retry` - ); - }); + test('should not retry initializing context specific resources if context specific resource initialization is in progress', async () => { + // this is the initial call that fails + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); - test('should not retry initializing common resources if common resource initialization is in progress', async () => { - // this is the initial call that fails - clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); - - // this is the retry call that we'll artificially inflate the duration of - clusterClient.cluster.putComponentTemplate.mockImplementationOnce(async () => { - await new Promise((r) => setTimeout(r, 1000)); - return { acknowledged: true }; - }); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - - // Installing ILM policy failed so no calls to install context-specific resources - // should be made - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - // call createAIAssistantConversationsDataClient at the same time which will trigger the retries - const result = await Promise.all([ - assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }), - assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }), - ]); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 2 - ); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).toHaveBeenCalled(); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).not.toHaveBeenCalled(); - expect(clusterClient.indices.getDataStream).toHaveBeenCalled(); - } else { - expect(clusterClient.indices.create).toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).toHaveBeenCalled(); - } - expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - dataStreamAdapter, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - kibanaVersion: '8.8.0', - }); - - expect(result[0]).not.toBe(null); - expect(result[1]).not.toBe(null); - expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.info).toHaveBeenCalledWith( - `Resource installation for "test" succeeded after retry` - ); - expect(logger.info).toHaveBeenCalledWith( - `Skipped retrying common resource initialization because it is already being retried.` - ); - }); + // this is the retry call that we'll artificially inflate the duration of + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => { + await new Promise((r) => setTimeout(r, 1000)); + return SimulateTemplateResponse; + }); - test('should retry initializing context specific resources if context specific resource initialization failed', async () => { - clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ - ...SimulateTemplateResponse, - template: { - ...SimulateTemplateResponse.template, - mappings: {}, - }, - })); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - const result = await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - dataStreamAdapter, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - kibanaVersion: '8.8.0', - }); - - expect(result).not.toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.info).toHaveBeenCalledWith( - `Resource installation for "test" succeeded after retry` - ); - }); + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); - test('should not retry initializing context specific resources if context specific resource initialization is in progress', async () => { - // this is the initial call that fails - clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ - ...SimulateTemplateResponse, - template: { - ...SimulateTemplateResponse.template, - mappings: {}, - }, - })); - - // this is the retry call that we'll artificially inflate the duration of - clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => { - await new Promise((r) => setTimeout(r, 1000)); - return SimulateTemplateResponse; - }); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - const createAlertsClientWithDelay = async (delayMs: number | null) => { - if (delayMs) { - await new Promise((r) => setTimeout(r, delayMs)); - } - - return await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - }; - - const result = await Promise.all([ - createAlertsClientWithDelay(null), - createAlertsClientWithDelay(1), - ]); - - expect(AIAssistantConversationsDataClient).toHaveBeenCalledTimes(2); - expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - dataStreamAdapter, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - kibanaVersion: '8.8.0', - }); - - expect(result[0]).not.toBe(null); - expect(result[1]).not.toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); - - // Should only log the retry once because the second call should - // leverage the outcome of the first retry - expect( - logger.info.mock.calls.filter( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` - ).length - ).toEqual(1); - expect( - logger.info.mock.calls.filter( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => - calls[0] === `Resource installation for "test" succeeded after retry` - ).length - ).toEqual(1); - }); + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); - test('should throttle retries of initializing context specific resources', async () => { - // this is the initial call that fails - clusterClient.indices.simulateTemplate.mockImplementation(async () => ({ - ...SimulateTemplateResponse, - template: { - ...SimulateTemplateResponse.template, - mappings: {}, - }, - })); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - const createAlertsClientWithDelay = async (delayMs: number | null) => { - if (delayMs) { - await new Promise((r) => setTimeout(r, delayMs)); - } - - return await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - }; - - await Promise.all([ - createAlertsClientWithDelay(null), - createAlertsClientWithDelay(1), - createAlertsClientWithDelay(2), - ]); - - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); - - // Should only log the retry once because the second and third retries should be throttled - expect( - logger.info.mock.calls.filter( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` - ).length - ).toEqual(1); - }); + const createAIAssistantDatastreamClientWithDelay = async (delayMs: number | null) => { + if (delayMs) { + await new Promise((r) => setTimeout(r, delayMs)); + } - test('should return null if retrying common resources initialization fails again', async () => { - let failCount = 0; - clusterClient.cluster.putComponentTemplate.mockImplementation(() => { - throw new Error(`fail ${++failCount}`); - }); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - - // Installing ILM policy failed so no calls to install context-specific resources - // should be made - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - const result = await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 2 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - expect(result).toBe(null); - expect(AlertsClient).not.toHaveBeenCalled(); - expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.warn).toHaveBeenCalledWith( - expect.stringMatching( - /There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation\. fail \d+; Error after retry: Failure during installation\. fail \d+/ - ) - ); + return assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, }); + }; - test('should return null if retrying common resources initialization fails again with same error', async () => { - clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('fail')); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - - expect(assistantService.isInitialized()).toEqual(false); - - // Installing component template failed so no calls to install context-specific resources - // should be made - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 1 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - const result = await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 0 : 2 - ); - expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); - expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); - expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); - expect(clusterClient.indices.create).not.toHaveBeenCalled(); - - expect(result).toBe(null); - expect(AlertsClient).not.toHaveBeenCalled(); - expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.warn).toHaveBeenCalledWith( - `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Retry failed with error: Failure during installation. fail` - ); - }); + const result = await Promise.all([ + createAIAssistantDatastreamClientWithDelay(null), + createAIAssistantDatastreamClientWithDelay(1), + ]); + + expect(AIAssistantConversationsDataClient).toHaveBeenCalledTimes(2); + expect(AIAssistantConversationsDataClient).toHaveBeenCalledWith({ + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '.kibana-elastic-ai-assistant-conversations', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + logger, + }); - test('should return null if retrying context specific initialization fails again', async () => { - clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ - ...SimulateTemplateResponse, - template: { - ...SimulateTemplateResponse.template, - mappings: {}, - }, - })); - clusterClient.indices.putIndexTemplate.mockRejectedValueOnce( - new Error('fail index template') - ); - - assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - assistantService.register(TestRegistrationContext); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - const result = await assistantService.createAlertsClient({ - logger, - ruleType: ruleTypeWithAlertDefinition, - namespace: 'default', - rule: { - consumer: 'bar', - executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', - id: '1', - name: 'rule-name', - parameters: { - bar: true, - }, - revision: 0, - spaceId: 'default', - tags: ['rule-', '-tags'], - }, - }); - - expect(AlertsClient).not.toHaveBeenCalled(); - expect(result).toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); - expect(logger.warn).toHaveBeenCalledWith( - `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping; Error after retry: Failure during installation. fail index template` - ); - }); + expect(result[0]).not.toBe(null); + expect(result[1]).not.toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + + // Should only log the retry once because the second call should + // leverage the outcome of the first retry + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + ).length + ).toEqual(1); + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => calls[0] === `Resource installation for "test" succeeded after retry` + ).length + ).toEqual(1); + }); + + test('should throttle retries of initializing context specific resources', async () => { + // this is the initial call that fails + clusterClient.indices.simulateTemplate.mockImplementation(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), }); - describe('retries', () => { - test('should retry adding ILM policy for transient ES errors', async () => { - if (useDataStreamForAlerts) return; - - clusterClient.ilm.putLifecycle - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(3); - }); + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); - test('should retry adding component template for transient ES errors', async () => { - clusterClient.cluster.putComponentTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); - }); + const createAIAssistantDatastreamClientWithDelay = async (delayMs: number | null) => { + if (delayMs) { + await new Promise((r) => setTimeout(r, delayMs)); + } - test('should retry updating index template for transient ES errors', async () => { - clusterClient.indices.putIndexTemplate - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - expect(assistantService.isInitialized()).toEqual(true); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + return assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, }); + }; - test('should retry updating index settings for existing indices for transient ES errors', async () => { - clusterClient.indices.putSettings - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(3); - } else { - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); - } - }); + await Promise.all([ + createAIAssistantDatastreamClientWithDelay(null), + createAIAssistantDatastreamClientWithDelay(1), + createAIAssistantDatastreamClientWithDelay(2), + ]); + + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + + // Should only log the retry once because the second and third retries should be throttled + expect( + logger.info.mock.calls.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + ).length + ).toEqual(1); + }); - test('should retry updating index mappings for existing indices for transient ES errors', async () => { - clusterClient.indices.putMapping - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes( - useDataStreamForAlerts ? 3 : 4 - ); - }); + test('should return null if retrying common resources initialization fails again', async () => { + let failCount = 0; + clusterClient.cluster.putComponentTemplate.mockImplementation(() => { + throw new Error(`fail ${++failCount}`); + }); - test('should retry creating concrete index for transient ES errors', async () => { - clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ - data_streams: [], - })); - clusterClient.indices.createDataStream - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ acknowledged: true }); - clusterClient.indices.create - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ index: 'index', shards_acknowledged: true, acknowledged: true }); - const assistantService = new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - dataStreamAdapter, - }); - - await retryUntil( - 'alert service initialized', - async () => assistantService.isInitialized() === true - ); - - assistantService.register(TestRegistrationContext); - await retryUntil( - 'context initialized', - async () => (await getContextInitialized(assistantService)) === true - ); - - if (useDataStreamForAlerts) { - expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); - } else { - expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); - } - }); + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), }); - describe('timeout', () => { - test('should short circuit initialization if timeout exceeded', async () => { - clusterClient.cluster.putComponentTemplate.mockImplementationOnce(async () => { - await new Promise((resolve) => setTimeout(resolve, 20)); - return { acknowledged: true }; - }); - new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - timeoutMs: 10, - dataStreamAdapter, - }); - - await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); - - expect(logger.error).toHaveBeenCalledWith(new Error(`Timeout: it took more than 10ms`)); - }); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); - test('should short circuit initialization if pluginStop$ signal received but not throw error', async () => { - pluginStop$.next(); - new AIAssistantService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - kibanaVersion: '8.8.0', - timeoutMs: 10, - dataStreamAdapter, - }); - - await retryUntil('debug logger called', async () => logger.debug.mock.calls.length > 0); - - expect(logger.debug).toHaveBeenCalledWith( - `Server is stopping; must stop all async operations` - ); - }); + expect(assistantService.isInitialized()).toEqual(false); + + // Installing ILM policy failed so no calls to install context-specific resources + // should be made + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + const result = await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); + + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + expect(result).toBe(null); + expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringMatching( + /There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation\. fail \d+; Error after retry: Failure during installation\. fail \d+/ + ) + ); + }); + + test('should return null if retrying common resources initialization fails again with same error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('fail')); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + + expect(assistantService.isInitialized()).toEqual(false); + + // Installing component template failed so no calls to install context-specific resources + // should be made + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + const result = await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); + + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + + expect(result).toBe(null); + expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "test"` + ); + expect(logger.warn).toHaveBeenCalledWith( + `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Retry failed with error: Failure during installation. fail` + ); + }); + + test('should return null if retrying space specific initialization fails again', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + clusterClient.indices.putIndexTemplate.mockRejectedValueOnce( + new Error('fail index template') + ); + + assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + + const result = await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); + + expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); + expect(result).toBe(null); + expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith( + `Retrying resource initialization for context "default"` + ); + expect(logger.warn).toHaveBeenCalledWith( + `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping; Error after retry: Failure during installation. fail index template` + ); + }); + }); + + describe('retries', () => { + test('should retry adding component template for transient ES errors', async () => { + clusterClient.cluster.putComponentTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index template for transient ES errors', async () => { + clusterClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + expect(assistantService.isInitialized()).toEqual(true); + + await retryUntil( + 'space resources initialized', + async () => (await getSpaceResourcesInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index settings for existing indices for transient ES errors', async () => { + clusterClient.indices.putSettings + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + + await retryUntil( + 'space resources initialized', + async () => (await getSpaceResourcesInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index mappings for existing indices for transient ES errors', async () => { + clusterClient.indices.putMapping + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), + }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + + await retryUntil( + 'space resources initialized', + async () => (await getSpaceResourcesInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(3); + }); + + test('should retry creating concrete index for transient ES errors', async () => { + clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ + data_streams: [], + })); + clusterClient.indices.createDataStream + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + clusterClient.indices.create + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ index: 'index', shards_acknowledged: true, acknowledged: true }); + const assistantService = new AIAssistantService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + kibanaVersion: '8.8.0', + taskManager: taskManagerMock.createSetup(), }); + + await retryUntil( + 'AI Assistant service initialized', + async () => assistantService.isInitialized() === true + ); + + await retryUntil( + 'space resources initialized', + async () => (await getSpaceResourcesInitialized(assistantService)) === true + ); + + expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); }); - } + }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts index 644a895693dcb..4dd02d67ee903 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts @@ -8,6 +8,10 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { ConversationDataWriter } from './conversations_data_writer'; +import { + getCreateConversationSchemaMock, + getUpdateConversationSchemaMock, +} from '../__mocks__/conversations_schema.mock'; describe('ConversationDataWriter', () => { describe('#bulk', () => { @@ -29,75 +33,46 @@ describe('ConversationDataWriter', () => { it('converts a list of conversations to an appropriate list of operations', async () => { await writer.bulk({ - conversationsToCreate: [], + conversationsToCreate: [ + getCreateConversationSchemaMock(), + getCreateConversationSchemaMock(), + ], conversationsToUpdate: [], - conversationsToDelete: ['1'], + conversationsToDelete: [], }); - const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; + const { docs_created: docsCreated } = (esClientMock.bulk as jest.Mock).mock.lastCall; - expect(operations).toMatchInlineSnapshot(` - Array [ - Object { - "create": Object { - "_index": "conversations-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - }, - Object { - "create": Object { - "_index": "conversations-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - }, - ] - `); + expect(docsCreated).toMatchInlineSnapshot(`undefined`); }); it('converts a list of mixed conversations operations to an appropriate list of operations', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); await writer.bulk({ - conversationsToCreate: [], - conversationsToUpdate: [], + conversationsToCreate: [getCreateConversationSchemaMock()], + conversationsToUpdate: [getUpdateConversationSchemaMock()], conversationsToDelete: ['1'], }); - const [{ operations }] = (esClientMock.bulk as jest.Mock).mock.lastCall; + const { + docs_created: docsCreated, + docs_deleted: docsDeleted, + docs_updated: docsUpdated, + } = (esClientMock.bulk as jest.Mock).mock.lastCall; - expect(operations).toMatchInlineSnapshot(` - Array [ - Object { - "create": Object { - "_index": "conversations-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - }, - Object { - "update": Object { - "_index": "conversations-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - }, - Object { - "delete": Object { - "_index": "conversations-default", - }, - }, - Object { - "@timestamp": "2023-02-15T00:15:19.231Z", - }, - ] - `); + expect(docsCreated).toMatchInlineSnapshot(`undefined`); + + expect(docsUpdated).toMatchInlineSnapshot(`undefined`); + + expect(docsDeleted).toMatchInlineSnapshot(`undefined`); }); it('returns an error if something went wrong', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); (esClientMock.bulk as jest.Mock).mockRejectedValue(new Error('something went wrong')); const { errors } = await writer.bulk({ @@ -106,10 +81,20 @@ describe('ConversationDataWriter', () => { conversationsToDelete: ['1'], }); - expect(errors).toEqual(['something went wrong']); + expect(errors).toEqual([ + { + conversation: { + id: '', + }, + message: 'something went wrong', + }, + ]); }); it('returns the time it took to write the conversations', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); (esClientMock.bulk as jest.Mock).mockResolvedValue({ took: 123, items: [], @@ -124,22 +109,28 @@ describe('ConversationDataWriter', () => { expect(took).toEqual(123); }); - it('returns the number of docs written', async () => { + it('returns the array of docs deleted', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); (esClientMock.bulk as jest.Mock).mockResolvedValue({ - items: [{ create: { status: 201 } }, { create: { status: 200 } }], + items: [{ delete: { status: 201 } }, { delete: { status: 200 } }], }); - const { docs_deleted: docsWritten } = await writer.bulk({ + const { docs_deleted: docsDeleted } = await writer.bulk({ conversationsToCreate: [], conversationsToUpdate: [], - conversationsToDelete: ['1'], + conversationsToDelete: ['1', '2'], }); - expect(docsWritten).toEqual(2); + expect(docsDeleted.length).toEqual(2); }); describe('when some documents failed to be written', () => { beforeEach(() => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); (esClientMock.bulk as jest.Mock).mockResolvedValue({ errors: true, items: [ @@ -150,13 +141,13 @@ describe('ConversationDataWriter', () => { }); it('returns the number of docs written', async () => { - const { docs_created: docsWritten } = await writer.bulk({ - conversationsToCreate: [], + const { docs_created: docsCreated } = await writer.bulk({ + conversationsToCreate: [getCreateConversationSchemaMock()], conversationsToUpdate: [], - conversationsToDelete: ['1'], + conversationsToDelete: [], }); - expect(docsWritten).toEqual(1); + expect(docsCreated.length).toEqual(1); }); it('returns the errors', async () => { @@ -166,14 +157,28 @@ describe('ConversationDataWriter', () => { conversationsToDelete: ['1'], }); - expect(errors).toEqual(['something went wrong']); + expect(errors).toEqual([ + { + conversation: { + id: undefined, + }, + message: 'something went wrong', + status: 500, + }, + ]); }); }); - describe('when there are no conversations to write', () => { + describe('when there are no conversations to update', () => { it('returns an appropriate response', async () => { const response = await writer.bulk({}); - expect(response).toEqual({ errors: [], docs_written: 0, took: 0 }); + expect(response).toEqual({ + errors: [], + docs_created: [], + docs_deleted: [], + docs_updated: [], + took: 0, + }); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index f3ae1e8ecc646..54f99e3d5059a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -17,8 +17,16 @@ import { transformToCreateScheme } from './create_conversation'; import { transformToUpdateScheme } from './update_conversation'; import { SearchEsConversationSchema } from './types'; +export interface BulkOperationError { + message: string; + status?: number; + conversation: { + id: string; + }; +} + interface WriterBulkResponse { - errors: string[]; + errors: BulkOperationError[]; docs_created: string[]; docs_deleted: string[]; docs_updated: string[]; @@ -65,13 +73,34 @@ export class ConversationDataWriter implements ConversationDataWriter { return { errors: errors ? items - .map( - (item) => - item.create?.error?.reason ?? - item.update?.error?.reason ?? - item.delete?.error?.reason + .map((item) => + item.create?.error + ? { + message: item.create.error?.reason, + status: item.create.status, + conversation: { + id: item.create._id, + }, + } + : item.update?.error + ? { + message: item.update.error?.reason, + status: item.update.status, + conversation: { + id: item.update._id, + }, + } + : item.delete?.error + ? { + message: item.delete?.error?.reason, + status: item.delete?.status, + conversation: { + id: item.delete?._id, + }, + } + : undefined ) - .filter((error): error is string => !!error) + .filter((e) => e !== undefined) : [], docs_created: items .filter((item) => item.create?.status === 201 || item.create?.status === 200) @@ -83,16 +112,23 @@ export class ConversationDataWriter implements ConversationDataWriter { .filter((item) => item.update?.status === 201 || item.update?.status === 200) .map((item) => item.update?._id ?? ''), took, - }; + } as WriterBulkResponse; } catch (e) { this.options.logger.error(`Error bulk actions for conversations: ${e.message}`); return { - errors: [`${e.message}`], + errors: [ + { + message: e.message, + conversation: { + id: '', + }, + }, + ], docs_created: [], docs_deleted: [], docs_updated: [], took: 0, - }; + } as WriterBulkResponse; } }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index 6b7a2da52a460..b6a037322f2ad 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -42,11 +42,14 @@ export const getConversationResponseMock = (): ConversationResponse => ({ messages: [], // eslint-disable-next-line @typescript-eslint/no-explicit-any replacements: {} as any, - createdAt: Date.now().toLocaleString(), - namespace: 'default', + createdAt: '2024-01-28T04:20:02.394Z', + namespace: 'test', isDefault: false, - updatedAt: Date.now().toLocaleString(), - timestamp: Date.now().toLocaleString(), + updatedAt: '2024-01-28T04:20:02.394Z', + timestamp: '2024-01-28T04:20:02.394Z', + user: { + name: 'test', + }, }); describe('createConversation', () => { @@ -58,6 +61,16 @@ describe('createConversation', () => { jest.clearAllMocks(); }); + beforeAll(() => { + jest.useFakeTimers(); + const date = '2024-01-28T04:20:02.394Z'; + jest.setSystemTime(new Date(date)); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + test('it returns a conversation as expected with the id changed out for the elastic id', async () => { const conversation = getCreateConversationMock(); @@ -85,7 +98,7 @@ describe('createConversation', () => { test('it returns a conversation as expected with the id changed out for the elastic id and title set', async () => { const conversation: ConversationCreateProps = { ...getCreateConversationMock(), - title: '{{value}}', + title: 'test new title', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 4d3da0eb0520b..d9ab986934283 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -72,12 +72,11 @@ export const createConversation = async ({ const createdAt = new Date().toISOString(); const body: CreateMessageSchema = transformToCreateScheme(createdAt, spaceId, user, conversation); - const response = await esClient.index({ + const response = await esClient.create({ body, id: uuidv4(), index: conversationIndex, refresh: 'wait_for', - op_type: 'create', }); return { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index d01e7134142a8..b1f117e4caecf 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -53,17 +53,17 @@ describe('deleteConversation', () => { test('Delete returns a null if the conversation is also null', async () => { (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); const options = getDeleteConversationOptionsMock(); - const deletedList = await deleteConversation(options); - expect(deletedList).toEqual(null); + const deletedConversation = await deleteConversation(options); + expect(deletedConversation).toEqual(null); }); - test('Delete returns the conversation if a conversation is returned from getConversation', async () => { + test('Delete returns the conversation id if a conversation is returned from getConversation', async () => { const conversation = getConversationResponseMock(); (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); const options = getDeleteConversationOptionsMock(); options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); - const deletedList = await deleteConversation(options); - expect(deletedList).toEqual(conversation); + const deletedConversationId = await deleteConversation(options); + expect(deletedConversationId).toEqual(conversation.id); }); test('Delete does not call data client if the conversation returns null', async () => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index c8eeec2e00e7b..f824b35172fbe 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -17,7 +17,7 @@ export const deleteConversation = async ({ esClient, conversationIndex, id, -}: DeleteConversationParams): Promise => { +}: DeleteConversationParams): Promise => { const conversation = await getConversation(esClient, conversationIndex, id); if (conversation !== null) { const response = await esClient.deleteByQuery({ @@ -36,5 +36,7 @@ export const deleteConversation = async ({ if (!response.deleted && response.deleted === 0) { throw Error('No conversation has been deleted'); } + return conversation.id ?? null; } + return null; }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index 7bc45138d3c3d..e21d2e994a342 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -18,6 +18,21 @@ export const getConversationResponseMock = (): ConversationResponse => ({ messages: [], id: '1', namespace: 'default', + isDefault: true, + excludeFromLastConversationStorage: false, + timestamp: '2020-04-20T15:25:31.830Z', + apiConfig: { + connectorId: 'c1', + connectorTypeTitle: 'title-c-1', + defaultSystemPromptId: 'prompt-1', + model: 'test', + provider: 'Azure OpenAI', + }, + user: { + id: '1111', + name: 'elastic', + }, + replacements: undefined, }); export const getSearchConversationMock = @@ -43,6 +58,20 @@ export const getSearchConversationMock = messages: [], id: '1', namespace: 'default', + is_default: true, + exclude_from_last_conversation_storage: false, + api_config: { + connector_id: 'c1', + connector_type_title: 'title-c-1', + default_system_prompt_id: 'prompt-1', + model: 'test', + provider: 'Azure OpenAI', + }, + user: { + id: '1111', + name: 'elastic', + }, + replacements: undefined, }, }, ], @@ -66,7 +95,11 @@ describe('getConversation', () => { const data = getSearchConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation(esClient, '', '1'); + const conversation = await getConversation( + esClient, + '.kibana-elastic-ai-assistant-conversations', + '1' + ); const expected = getConversationResponseMock(); expect(conversation).toEqual(expected); }); @@ -76,7 +109,11 @@ describe('getConversation', () => { data.hits.hits = []; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation(esClient, '', '1'); + const conversation = await getConversation( + esClient, + '.kibana-elastic-ai-assistant-conversations', + '1' + ); expect(conversation).toEqual(null); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts index 5d18ed8bfa0f6..a4f60c3d34314 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts @@ -4,28 +4,219 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import type { UpdateByQueryRequest } from '@elastic/elasticsearch/lib/api/types'; +import { AIAssistantConversationsDataClient, AIAssistantConversationsDataClientParams } from '.'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; +import { + getConversationMock, + getQueryConversationParams, + getUpdateConversationSchemaMock, +} from '../__mocks__/conversations_schema.mock'; -import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; -import { LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; +const date = '2023-03-28T22:27:28.159Z'; +let logger: ReturnType; +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; -import { getListClientMock } from './list_client.mock'; +const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, +} as AuthenticatedUser; describe('AIAssistantConversationsDataClient', () => { - describe('Mock client checks (not exhaustive tests against it)', () => { - test('it returns the get list index as expected', () => { - const mock = getListClientMock(); - expect(mock.getListName()).toEqual(LIST_INDEX); + let assistantConversationsDataClientParams: AIAssistantConversationsDataClientParams; + + beforeEach(() => { + jest.clearAllMocks(); + logger = loggingSystemMock.createLogger(); + assistantConversationsDataClientParams = { + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + spaceId: 'default', + indexPatternsResorceName: '', + currentUser: mockUser1, + kibanaVersion: '8.8.0', + }; + }); + + beforeAll(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date(date)); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + test('should get by id the persistent conversation successfully', async () => { + clusterClient.search.mockReturnValue({ + // @ts-ignore + hits: { + total: { value: 1 }, + hits: [ + { + _source: { + '@timestamp': '2024-01-25T01:32:37.649Z', + updated_at: '2024-01-25T01:34:51.303Z', + api_config: { + connector_id: 'bedbf764-b991-4115-a9fc-1cfeaef21046', + model: 'anthropic.claude-v2', + connector_type_title: 'Amazon Bedrock', + }, + namespace: 'hghjghjghghjghg33', + created_at: '2024-01-25T01:32:37.649Z', + messages: [ + { + presentation: { + delay: 1000, + stream: true, + }, + '@timestamp': '1/24/2024, 5:32:19 PM', + role: 'assistant', + reader: null, + is_error: null, + replacements: null, + content: + 'Go ahead and click the add connector button below to continue the conversation!', + }, + { + presentation: null, + '@timestamp': '1/24/2024, 5:32:37 PM', + role: 'assistant', + reader: null, + is_error: null, + replacements: null, + content: 'Connector setup complete!', + }, + { + presentation: null, + '@timestamp': '1/24/2024, 5:34:50 PM', + role: 'assistant', + reader: null, + is_error: true, + replacements: null, + content: 'An error occurred sending your message.', + }, + ], + title: 'Alert summary', + is_default: true, + user: { + name: 'elastic', + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + }, + }, + }, + ], + }, }); - test('it returns the get list item index as expected', () => { - const mock = getListClientMock(); - expect(mock.getListItemName()).toEqual(LIST_ITEM_INDEX); + const assistantConversationsDataClient = new AIAssistantConversationsDataClient( + assistantConversationsDataClientParams + ); + const result = await assistantConversationsDataClient.getConversation('1'); + + expect(clusterClient.search).toHaveBeenCalledTimes(1); + + expect(result).toEqual({ + apiConfig: { + connectorId: 'bedbf764-b991-4115-a9fc-1cfeaef21046', + connectorTypeTitle: 'Amazon Bedrock', + defaultSystemPromptId: undefined, + model: 'anthropic.claude-v2', + provider: undefined, + }, + createdAt: '2024-01-25T01:32:37.649Z', + excludeFromLastConversationStorage: undefined, + id: undefined, + isDefault: true, + messages: [ + { + content: + 'Go ahead and click the add connector button below to continue the conversation!', + presentation: { + delay: 1000, + stream: true, + }, + role: 'assistant', + timestamp: '1/24/2024, 5:32:19 PM', + }, + { + content: 'Connector setup complete!', + role: 'assistant', + timestamp: '1/24/2024, 5:32:37 PM', + }, + { + content: 'An error occurred sending your message.', + isError: true, + role: 'assistant', + timestamp: '1/24/2024, 5:34:50 PM', + }, + ], + namespace: 'hghjghjghghjghg33', + replacements: undefined, + timestamp: '2024-01-25T01:32:37.649Z', + title: 'Alert summary', + updatedAt: '2024-01-25T01:34:51.303Z', + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, + }); + }); + + test('should update conversation with new messages', async () => { + const assistantConversationsDataClient = new AIAssistantConversationsDataClient( + assistantConversationsDataClientParams + ); + + await assistantConversationsDataClient.updateConversation( + getConversationMock(getQueryConversationParams()), + getUpdateConversationSchemaMock('123345') + ); + + const params = clusterClient.updateByQuery.mock.calls[0][0] as UpdateByQueryRequest; + + expect(params.query).toEqual({ + ids: { + values: ['04128c15-0d1b-4716-a4c5-46997ac7f3bd'], + }, }); - test('it returns a mock list item', async () => { - const mock = getListClientMock(); - const listItem = await mock.getListItem({ id: '123' }); - expect(listItem).toEqual(getListItemResponseMock()); + expect(params.script).toEqual({ + source: expect.anything(), + lang: 'painless', + params: { + api_config: { + connector_id: '2', + connector_type_title: 'Test connector', + default_system_prompt_id: 'Default', + model: 'model', + provider: undefined, + }, + assignEmpty: false, + exclude_from_last_conversation_storage: false, + messages: [ + { + '@timestamp': '2019-12-13T16:40:33.400Z', + content: 'test content', + is_error: undefined, + presentation: undefined, + reader: undefined, + replacements: undefined, + role: 'user', + trace_data: { + trace_id: '1', + transaction_id: '2', + }, + }, + ], + replacements: undefined, + title: 'Welcome 2', + updated_at: '2023-03-28T22:27:28.159Z', + }, }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 1a2620ae2bc84..29d1d4b2d94c4 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -62,14 +62,14 @@ export class AIAssistantConversationsDataClient { this.spaceId = this.options.spaceId; } - public async getWriter(): Promise { + public getWriter = async (): Promise => { const spaceId = this.spaceId; if (this.writerCache.get(spaceId)) { return this.writerCache.get(spaceId) as ConversationDataWriter; } await this.initializeWriter(spaceId, this.indexTemplateAndPattern.alias); return this.writerCache.get(spaceId) as ConversationDataWriter; - } + }; private async initializeWriter(spaceId: string, index: string): Promise { const esClient = await this.options.elasticsearchClientPromise; @@ -85,7 +85,7 @@ export class AIAssistantConversationsDataClient { return writer; } - public getReader(options: { spaceId?: string } = {}) { + public getReader = async (options: { spaceId?: string } = {}) => { const indexPatterns = this.indexTemplateAndPattern.alias; return { @@ -111,7 +111,7 @@ export class AIAssistantConversationsDataClient { } }, }; - } + }; public getConversation = async (id: string): Promise => { const esClient = await this.options.elasticsearchClientPromise; @@ -229,6 +229,10 @@ export class AIAssistantConversationsDataClient { */ public deleteConversation = async (id: string): Promise => { const esClient = await this.options.elasticsearchClientPromise; - deleteConversation({ esClient, conversationIndex: this.indexTemplateAndPattern.alias, id }); + await deleteConversation({ + esClient, + conversationIndex: this.indexTemplateAndPattern.alias, + id, + }); }; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 59429af9af453..60d645098e30a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -44,11 +44,11 @@ export const getConversationResponseMock = (): ConversationResponse => ({ messages: [], // eslint-disable-next-line @typescript-eslint/no-explicit-any replacements: {} as any, - createdAt: Date.now().toLocaleString(), + createdAt: '2020-04-20T15:25:31.830Z', namespace: 'default', isDefault: false, - updatedAt: Date.now().toLocaleString(), - timestamp: Date.now().toLocaleString(), + updatedAt: '2020-04-20T15:25:31.830Z', + timestamp: '2020-04-20T15:25:31.830Z', }); jest.mock('./get_conversation', () => ({ diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 4a863d1a4f701..e773f9e9779ac 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -136,6 +136,10 @@ export const updateConversation = async ({ ); return null; } + + if (!response.updated && response.updated === 0) { + throw Error('No conversation has been updated'); + } } catch (err) { logger.warn(`Error updating conversation: ${err} by ID: ${existingConversation.id}`); throw err; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts new file mode 100644 index 0000000000000..f60c354e6f21c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES } from '@kbn/elastic-assistant-common'; +import { getAppendConversationMessageRequest, requestMock } from '../../__mocks__/request'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { serverMock } from '../../__mocks__/server'; +import { + getAppendConversationMessagesSchemaMock, + getConversationMock, + getQueryConversationParams, + getUpdateConversationSchemaMock, +} from '../../__mocks__/conversations_schema.mock'; +import { appendConversationMessageRoute } from './append_conversation_messages_route'; + +describe('Append conversation messages route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); + clients.elasticAssistant.getAIAssistantConversationsDataClient.appendConversationMessages.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); // successful append + + appendConversationMessageRoute(server.router); + }); + + describe('status codes', () => { + test('returns 200', async () => { + const response = await server.inject( + getAppendConversationMessageRequest('04128c15-0d1b-4716-a4c5-46997ac7f3bd'), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(200); + }); + + test('returns 404 when append to a conversation that does not exist', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + null + ); + + const response = await server.inject( + getAppendConversationMessageRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(404); + expect(response.body).toEqual({ + message: 'conversation id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" not found', + status_code: 404, + }); + }); + + test('catches error if search throws error', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockImplementation( + async () => { + throw new Error('Test error'); + } + ); + const response = await server.inject( + getAppendConversationMessageRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + }); + + describe('request validation', () => { + test('rejects payloads with no ID', async () => { + const noIdRequest = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, + body: { + ...getAppendConversationMessagesSchemaMock(), + id: undefined, + }, + }); + const response = await server.validate(noIdRequest); + expect(response.badRequest).toHaveBeenCalled(); + }); + + test('allows messages only', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, + body: { + ...getAppendConversationMessagesSchemaMock(), + apiConfig: { + defaultSystemPromptId: 'test', + }, + }, + params: { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd' }, + }); + const result = server.validate(request); + + expect(result.ok).toHaveBeenCalled(); + }); + + test('disallows invalid message "role" value', async () => { + const request = requestMock.create({ + method: 'post', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, + body: { + ...getUpdateConversationSchemaMock(), + messages: [ + { + role: 'invalid', + content: 'test', + timestamp: '2019-12-13T16:40:33.400Z', + }, + ], + }, + }); + const result = server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + `messages.0.role: Invalid enum value. Expected 'system' | 'user' | 'assistant', received 'invalid'` + ); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts index eca60c5ca2db7..537bcb49fda06 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts @@ -11,11 +11,12 @@ import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getConversationsBulkActionRequest, requestMock } from '../../__mocks__/request'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION } from '@kbn/elastic-assistant-common'; +import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; import { - getEmptyFindResult, - getFindConversationsResultWithSingleHit, -} from '../../__mocks__/response'; -import { getPerformBulkActionSchemaMock } from '../../__mocks__/conversations_schema.mock'; + getCreateConversationSchemaMock, + getPerformBulkActionSchemaMock, + getUpdateConversationSchemaMock, +} from '../../__mocks__/conversations_schema.mock'; describe('Perform bulk action route', () => { let server: ReturnType; @@ -23,7 +24,7 @@ describe('Perform bulk action route', () => { let logger: ReturnType; const mockConversation = getFindConversationsResultWithSingleHit().data[0]; - beforeEach(() => { + beforeEach(async () => { server = serverMock.create(); logger = loggingSystemMock.createLogger(); ({ clients, context } = requestContextMock.createTools()); @@ -31,83 +32,77 @@ describe('Perform bulk action route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( getFindConversationsResultWithSingleHit() ); + ( + (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) + .bulk as jest.Mock + ).mockResolvedValue({ + docs_created: [mockConversation, mockConversation], + docs_updated: [mockConversation, mockConversation], + docs_deleted: [], + errors: [], + }); bulkActionConversationsRoute(server.router, logger); }); describe('status codes', () => { it('returns 200 when performing bulk action with all dependencies present', async () => { const response = await server.inject( - getConversationsBulkActionRequest(), - requestContextMock.convertContext(context) - ); - expect(response.status).toEqual(200); - expect(response.body).toEqual({ - success: true, - rules_count: 1, - attributes: { - results: someBulkActionResults(), - summary: { - failed: 0, - skipped: 0, - succeeded: 1, - total: 1, - }, - }, - }); - }); - - it("returns 200 when provided filter query doesn't match any conversations", async () => { - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getEmptyFindResult() - ); - const response = await server.inject( - getConversationsBulkActionRequest(), + getConversationsBulkActionRequest( + [getCreateConversationSchemaMock()], + [getUpdateConversationSchemaMock()], + ['99403909-ca9b-49ba-9d7a-7e5320e68d05'] + ), requestContextMock.convertContext(context) ); expect(response.status).toEqual(200); expect(response.body).toEqual({ success: true, - rules_count: 0, + conversations_count: 2, attributes: { results: someBulkActionResults(), summary: { failed: 0, skipped: 0, - succeeded: 0, - total: 0, + succeeded: 2, + total: 2, }, }, }); }); }); - describe('rules execution failures', () => { - it('returns partial failure error if update of few rules fail', async () => { + describe('conversations bulk actions failures', () => { + it('returns partial failure error if update of few conversations fail', async () => { ( (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) .bulk as jest.Mock ).mockResolvedValue({ - rules: [mockConversation, mockConversation], - skipped: [], + docs_created: [mockConversation, mockConversation], + docs_updated: [], + docs_deleted: [], errors: [ { message: 'mocked validation message', - conversation: { id: 'failed-conversation-id-1', name: 'Detect Root/Admin Users' }, + conversation: { id: 'failed-conversation-id-1', title: 'Detect Root/Admin Users' }, }, { message: 'mocked validation message', - conversation: { id: 'failed-conversation-id-2', name: 'Detect Root/Admin Users' }, + conversation: { id: 'failed-conversation-id-2', title: 'Detect Root/Admin Users' }, }, { message: 'test failure', - conversation: { id: 'failed-conversation-id-3', name: 'Detect Root/Admin Users' }, + conversation: { id: 'failed-conversation-id-3', title: 'Detect Root/Admin Users' }, }, ], total: 5, }); const response = await server.inject( - getConversationsBulkActionRequest(), + getConversationsBulkActionRequest( + [getCreateConversationSchemaMock()], + [getUpdateConversationSchemaMock()], + ['99403909-ca9b-49ba-9d7a-7e5320e68d05'] + ), requestContextMock.convertContext(context) ); @@ -125,76 +120,28 @@ describe('Perform bulk action route', () => { message: 'mocked validation message', conversations: [ { - id: 'failed-rule-id-1', - name: 'Detect Root/Admin Users', - }, - { - id: 'failed-rule-id-2', - name: 'Detect Root/Admin Users', + id: 'failed-conversation-id-1', + name: '', }, ], status_code: 500, }, { - message: 'test failure', - rules: [ + message: 'mocked validation message', + conversations: [ { - id: 'failed-rule-id-3', - name: 'Detect Root/Admin Users', + id: 'failed-conversation-id-2', + name: '', }, ], status_code: 500, }, - ], - results: someBulkActionResults(), - }, - message: 'Bulk edit partially failed', - status_code: 500, - }); - }); - }); - - describe('conversation skipping', () => { - it('returns partial failure error with skipped rules if some rule updates fail and others are skipped', async () => { - ( - (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) - .bulk as jest.Mock - ).mockResolvedValue({ - rules: [mockConversation, mockConversation], - skipped: [ - { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, - { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, - ], - errors: [ - { - message: 'test failure', - rule: { id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }, - }, - ], - total: 5, - }); - - const response = await server.inject( - getConversationsBulkActionRequest(), - requestContextMock.convertContext(context) - ); - - expect(response.status).toEqual(500); - expect(response.body).toEqual({ - attributes: { - summary: { - failed: 1, - skipped: 2, - succeeded: 2, - total: 5, - }, - errors: [ { message: 'test failure', - rules: [ + conversations: [ { - id: 'failed-rule-id-3', - name: 'Detect Root/Admin Users', + id: 'failed-conversation-id-3', + name: '', }, ], status_code: 500, @@ -203,109 +150,26 @@ describe('Perform bulk action route', () => { results: someBulkActionResults(), }, message: 'Bulk edit partially failed', - status_code: 500, - }); - }); - - it('returns success with skipped rules if some rules are skipped, but no errors are reported', async () => { - ( - (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) - .bulk as jest.Mock - ).mockResolvedValue({ - rules: [mockConversation, mockConversation], - skipped: [ - { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, - { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, - ], - errors: [], - total: 4, - }); - - const response = await server.inject( - getConversationsBulkActionRequest(), - requestContextMock.convertContext(context) - ); - - expect(response.status).toEqual(200); - expect(response.body).toEqual({ - attributes: { - summary: { - failed: 0, - skipped: 2, - succeeded: 2, - total: 4, - }, - results: someBulkActionResults(), - }, - rules_count: 4, - success: true, - }); - }); - - it('returns 500 with skipped rules if some rules are skipped, but some errors are reported', async () => { - ( - (await clients.elasticAssistant.getAIAssistantConversationsDataClient.getWriter()) - .bulk as jest.Mock - ).mockResolvedValue({ - rules: [mockConversation, mockConversation], - skipped: [ - { id: 'skipped-rule-id-1', name: 'Skipped Rule 1', skip_reason: 'RULE_NOT_MODIFIED' }, - { id: 'skipped-rule-id-2', name: 'Skipped Rule 2', skip_reason: 'RULE_NOT_MODIFIED' }, - ], - errors: [ - { - message: 'test failure', - rule: { id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }, - }, - ], - total: 5, - }); - - const response = await server.inject( - getConversationsBulkActionRequest(), - requestContextMock.convertContext(context) - ); - - expect(response.status).toEqual(500); - expect(response.body).toEqual({ - attributes: { - summary: { - failed: 1, - skipped: 2, - succeeded: 2, - total: 5, - }, - results: someBulkActionResults(), - errors: [ - { - message: 'test failure', - rules: [{ id: 'failed-rule-id-3', name: 'Detect Root/Admin Users' }], - status_code: 500, - }, - ], - }, - message: 'Bulk edit partially failed', - status_code: 500, }); }); }); describe('request validation', () => { - it('rejects payloads with no operations', async () => { + it('rejects payloads with no ids in delete operation', async () => { const request = requestMock.create({ - method: 'patch', + method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, - body: { ...getPerformBulkActionSchemaMock(), action: undefined }, + body: { ...getPerformBulkActionSchemaMock(), delete: { ids: [] } }, }); const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'action: Invalid literal value, expected "delete", action: Invalid literal value, expected "disable", action: Invalid literal value, expected "enable", action: Invalid literal value, expected "export", action: Invalid literal value, expected "duplicate", and 2 more' + 'delete.ids: Array must contain at least 1 element(s)' ); }); it('accepts payloads with only delete action', async () => { const request = requestMock.create({ - method: 'patch', + method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, body: getPerformBulkActionSchemaMock(), }); @@ -316,7 +180,7 @@ describe('Perform bulk action route', () => { it('accepts payloads with all operations', async () => { const request = requestMock.create({ - method: 'patch', + method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, body: getPerformBulkActionSchemaMock(), }); @@ -325,20 +189,20 @@ describe('Perform bulk action route', () => { expect(result.ok).toHaveBeenCalled(); }); - it('rejects payload if there is more than 100 updates in payload', async () => { + it('rejects payload if there is more than 100 deletes in payload', async () => { const request = requestMock.create({ - method: 'patch', + method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, body: { ...getPerformBulkActionSchemaMock(), - ids: Array.from({ length: 101 }).map(() => 'fake-id'), + delete: { ids: Array.from({ length: 101 }).map(() => 'fake-id') }, }, }); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(400); - expect(response.body.message).toEqual('More than 100 operations sent for bulk edit action.'); + expect(response.body.message).toEqual('More than 100 ids sent for bulk edit action.'); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts index 473e7d57e12f5..e7e61244ec5a1 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts @@ -33,7 +33,7 @@ export interface BulkOperationError { status?: number; conversation: { id: string; - name: string; + name?: string; }; } @@ -48,7 +48,7 @@ const buildBulkResponse = ( deleted = [], skipped = [], }: { - errors?: BulkActionError[]; + errors?: BulkOperationError[]; updated?: ConversationResponse[]; created?: ConversationResponse[]; deleted?: string[]; @@ -78,9 +78,14 @@ const buildBulkResponse = ( headers: { 'content-type': 'application/json' }, body: { message: summary.succeeded > 0 ? 'Bulk edit partially failed' : 'Bulk edit failed', - status_code: 500, + // status_code: 500, attributes: { - errors: [], + errors: errors.map((e: BulkOperationError) => ({ + status_code: e.status ?? 500, + conversations: [{ id: e.conversation.id, name: '' }], + message: e.message, + // err_code: '500', + })), results, summary, }, @@ -126,7 +131,11 @@ export const bulkActionConversationsRoute = ( const { body } = request; const assistantResponse = buildResponse(response); - if (body?.update && body.update?.length > CONVERSATIONS_TABLE_MAX_PAGE_SIZE) { + const operationsCount = + (body?.update ? body.update?.length : 0) + + (body?.create ? body.create?.length : 0) + + (body?.delete ? body.delete?.ids?.length ?? 0 : 0); + if (operationsCount > CONVERSATIONS_TABLE_MAX_PAGE_SIZE) { return assistantResponse.error({ body: `More than ${CONVERSATIONS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, statusCode: 400, @@ -143,6 +152,7 @@ export const bulkActionConversationsRoute = ( const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const writer = await dataClient?.getWriter(); + const { errors, docs_created: docsCreated, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts index 718eb1ca81669..e0a03497acf85 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts @@ -89,27 +89,13 @@ describe('Create conversation route', () => { }); describe('request validation', () => { - test('allows rule type of query', async () => { + test('disallows unknown title', async () => { const request = requestMock.create({ method: 'post', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, body: { ...getCreateConversationSchemaMock(), - type: 'query', - }, - }); - const result = server.validate(request); - - expect(result.ok).toHaveBeenCalled(); - }); - - test('disallows unknown rule type', async () => { - const request = requestMock.create({ - method: 'post', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, - body: { - ...getCreateConversationSchemaMock(), - type: 'unexpected_type', + title: true, }, }); const result = server.validate(request); @@ -118,14 +104,12 @@ describe('Create conversation route', () => { }); }); describe('conversation containing messages', () => { - const getResponseAction = (command: string = 'isolate') => ({ - role: 'user', - params: { - command, - comment: '', - }, + const getMessage = (role: string = 'user') => ({ + role, + content: 'test content', + timestamp: '2019-12-13T16:40:33.400Z', }); - const defaultAction = getResponseAction(); + const defaultMessage = getMessage(); test('is successful', async () => { const request = requestMock.create({ @@ -133,7 +117,7 @@ describe('Create conversation route', () => { path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, body: { ...getCreateConversationSchemaMock(), - response_actions: [defaultAction], + messages: [defaultMessage], }, }); @@ -142,7 +126,7 @@ describe('Create conversation route', () => { }); test('fails when provided with an unsupported message role', async () => { - const wrongMessage = getResponseAction('test_thing'); + const wrongMessage = getMessage('test_thing'); const request = requestMock.create({ method: 'post', @@ -154,7 +138,7 @@ describe('Create conversation route', () => { }); const result = await server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'messages.0.role: Invalid literal value, expected "user", messages.0.params.command: Invalid literal value, expected "isolate"' + `messages.0.role: Invalid enum value. Expected 'system' | 'user' | 'assistant', received 'test_thing'` ); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts index 28d5075022d97..ab92221134fa3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts @@ -44,6 +44,21 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); + const currentUser = ctx.elasticAssistant.getCurrentUser(); + + const additionalFilter = `title:${request.body.title}`; + const result = await dataClient?.findConversations({ + perPage: 100, + page: 1, + filter: `user.id:${currentUser?.profile_uid}${additionalFilter}`, + fields: ['title'], + }); + if (result?.data != null && result.data.length > 0) { + return assistantResponse.error({ + statusCode: 409, + body: `conversation title: "${request.body.title}" already exists`, + }); + } const createdConversation = await dataClient?.createConversation(request.body); return response.ok({ body: ConversationResponse.parse(createdConversation), diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts index 0dec9a69e3de0..78fc56106b289 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts @@ -10,10 +10,11 @@ import { requestContextMock } from '../../__mocks__/request_context'; import { serverMock } from '../../__mocks__/server'; import { deleteConversationRoute } from './delete_route'; import { getDeleteConversationRequest, requestMock } from '../../__mocks__/request'; + import { - getEmptyFindResult, - getFindConversationsResultWithSingleHit, -} from '../../__mocks__/response'; + getConversationMock, + getQueryConversationParams, +} from '../../__mocks__/conversations_schema.mock'; describe('Delete conversation route', () => { let server: ReturnType; @@ -23,14 +24,17 @@ describe('Delete conversation route', () => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getFindConversationsResultWithSingleHit() + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) ); deleteConversationRoute(server.router); }); describe('status codes with getAIAssistantConversationsDataClient', () => { test('returns 200 when deleting a single conversation with a valid getAIAssistantConversationsDataClient by Id', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) + ); const response = await server.inject( getDeleteConversationRequest(), requestContextMock.convertContext(context) @@ -40,8 +44,8 @@ describe('Delete conversation route', () => { }); test('returns 404 when deleting a single rule that does not exist with a valid actionClient and alertClient', async () => { - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getEmptyFindResult() + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + null ); const response = await server.inject( @@ -51,7 +55,7 @@ describe('Delete conversation route', () => { expect(response.status).toEqual(404); expect(response.body).toEqual({ - message: 'conversation id: "conversation-1" not found', + message: 'conversation id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" not found', status_code: 404, }); }); @@ -81,12 +85,9 @@ describe('Delete conversation route', () => { path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, query: {}, }); - const response = await server.inject(request, requestContextMock.convertContext(context)); - expect(response.status).toEqual(400); - expect(response.body).toEqual({ - message: ['either "id" or "rule_id" must be set'], - status_code: 400, - }); + const result = server.validate(request); + + expect(result.badRequest).toHaveBeenCalled(); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts index c2677a6173ccf..99d0215bf09ef 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts @@ -12,10 +12,6 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND } from '@kbn/elastic-assist import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; -import { - getConversationMock, - getQueryConversationParams, -} from '../../__mocks__/conversations_schema.mock'; describe('Find conversations route', () => { let server: ReturnType; @@ -30,9 +26,6 @@ describe('Find conversations route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( getFindConversationsResultWithSingleHit() ); - clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( - getConversationMock(getQueryConversationParams()) - ); findConversationsRoute(server.router, logger); }); @@ -72,7 +65,7 @@ describe('Find conversations route', () => { query: { page: 2, per_page: 20, - sort_field: 'name', + sort_field: 'title', fields: ['field1', 'field2'], }, }); @@ -81,6 +74,24 @@ describe('Find conversations route', () => { expect(result.ok).toHaveBeenCalled(); }); + test('disallows invalid sort fields', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, + query: { + page: 2, + per_page: 20, + sort_field: 'name', + fields: ['field1', 'field2'], + }, + }); + const result = server.validate(request); + + expect(result.badRequest).toHaveBeenCalledWith( + `sort_field: Invalid enum value. Expected 'created_at' | 'is_default' | 'title' | 'updated_at', received 'name'` + ); + }); + test('ignores unknown query params', async () => { const request = requestMock.create({ method: 'get', diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts index b24c439f20ae4..3025b734bc593 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts @@ -5,16 +5,12 @@ * 2.0. */ -import { getFindRequest, requestMock } from '../../__mocks__/request'; +import { getCurrentUserFindRequest, requestMock } from '../../__mocks__/request'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS } from '@kbn/elastic-assistant-common'; import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; import { findUserConversationsRoute } from './find_user_conversations_route'; -import { - getConversationMock, - getQueryConversationParams, -} from '../../__mocks__/conversations_schema.mock'; describe('Find user conversations route', () => { let server: ReturnType; @@ -27,9 +23,13 @@ describe('Find user conversations route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( getFindConversationsResultWithSingleHit() ); - clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( - getConversationMock(getQueryConversationParams()) - ); + clients.elasticAssistant.getCurrentUser.mockResolvedValue({ + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + }); findUserConversationsRoute(server.router); }); @@ -37,7 +37,7 @@ describe('Find user conversations route', () => { describe('status codes', () => { test('returns 200', async () => { const response = await server.inject( - getFindRequest(), + getCurrentUserFindRequest(), requestContextMock.convertContext(context) ); expect(response.status).toEqual(200); @@ -50,7 +50,7 @@ describe('Find user conversations route', () => { } ); const response = await server.inject( - getFindRequest(), + getCurrentUserFindRequest(), requestContextMock.convertContext(context) ); expect(response.status).toEqual(500); @@ -69,7 +69,7 @@ describe('Find user conversations route', () => { query: { page: 2, per_page: 20, - sort_field: 'name', + sort_field: 'title', fields: ['field1', 'field2'], }, }); @@ -78,6 +78,24 @@ describe('Find user conversations route', () => { expect(result.ok).toHaveBeenCalled(); }); + test('disallows invalid sort fields', async () => { + const request = requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + query: { + page: 2, + per_page: 20, + sort_field: 'name', + fields: ['field1', 'field2'], + }, + }); + const result = server.validate(request); + + expect(result.badRequest).toHaveBeenCalledWith( + `sort_field: Invalid enum value. Expected 'created_at' | 'is_default' | 'title' | 'updated_at', received 'name'` + ); + }); + test('ignores unknown query params', async () => { const request = requestMock.create({ method: 'get', diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts index 7d1027c5d82fb..d34fa7af9ea0b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts @@ -8,12 +8,11 @@ import { requestContextMock } from '../../__mocks__/request_context'; import { serverMock } from '../../__mocks__/server'; import { readConversationRoute } from './read_route'; -import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; +import { getConversationReadRequest, requestMock } from '../../__mocks__/request'; import { - getConversationReadRequest, - getConversationReadRequestWithId, - requestMock, -} from '../../__mocks__/request'; + getConversationMock, + getQueryConversationParams, +} from '../../__mocks__/conversations_schema.mock'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assistant-common'; describe('Read conversation route', () => { @@ -25,8 +24,8 @@ describe('Read conversation route', () => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getFindConversationsResultWithSingleHit() + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + getConversationMock(getQueryConversationParams()) ); readConversationRoute(server.router); }); @@ -40,9 +39,9 @@ describe('Read conversation route', () => { expect(response.status).toEqual(200); }); - test('returns 200 when reading a single rule outcome === exactMatch', async () => { + test('returns 200 when reading a single conversation outcome === exactMatch', async () => { const response = await server.inject( - getConversationReadRequestWithId(myFakeId), + getConversationReadRequest(myFakeId), requestContextMock.convertContext(context) ); expect(response.status).toEqual(200); @@ -69,17 +68,20 @@ describe('Read conversation route', () => { describe('data validation', () => { test('returns 404 if given a non-existent id', async () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( - {} + null ); const request = requestMock.create({ method: 'get', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - query: { id: 'DNE_RULE' }, + params: { id: '99403909-ca9b-49ba-9d7a-7e5320e68d05' }, }); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(404); - expect(response.body).toEqual({ message: 'rule_id: "DNE_RULE" not found', status_code: 404 }); + expect(response.body).toEqual({ + message: 'conversation id: "99403909-ca9b-49ba-9d7a-7e5320e68d05" not found', + status_code: 404, + }); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts index 143c6036c0bd2..a6701f7c35ea7 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts @@ -36,7 +36,7 @@ export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { }, }, async (context, request, response): Promise> => { - const responseObj = buildResponse(response); + const assistantResponse = buildResponse(response); const { id } = request.params; @@ -45,10 +45,17 @@ export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const conversation = await dataClient?.getConversation(id); - return response.ok({ body: conversation ?? {} }); + + if (conversation == null) { + return assistantResponse.error({ + body: `conversation id: "${id}" not found`, + statusCode: 404, + }); + } + return response.ok({ body: conversation }); } catch (err) { const error = transformError(err); - return responseObj.error({ + return assistantResponse.error({ body: error.message, statusCode: error.statusCode, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts index 48e0139204459..d4c47f6302abb 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts @@ -13,7 +13,6 @@ import { getQueryConversationParams, getUpdateConversationSchemaMock, } from '../../__mocks__/conversations_schema.mock'; -import { getEmptyFindResult } from '../../__mocks__/response'; import { updateConversationRoute } from './update_route'; describe('Update conversation route', () => { @@ -24,12 +23,9 @@ describe('Update conversation route', () => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getEmptyFindResult() - ); // no current conversations - clients.elasticAssistant.getAIAssistantConversationsDataClient.createConversation.mockResolvedValue( + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( getConversationMock(getQueryConversationParams()) - ); // creation succeeds + ); clients.elasticAssistant.getAIAssistantConversationsDataClient.updateConversation.mockResolvedValue( getConversationMock(getQueryConversationParams()) ); // successful update @@ -46,9 +42,9 @@ describe('Update conversation route', () => { expect(response.status).toEqual(200); }); - test('returns 404 when updating a single rule that does not exist', async () => { - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getEmptyFindResult() + test('returns 404 when updating a single conversation that does not exist', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( + null ); const response = await server.inject( @@ -58,13 +54,13 @@ describe('Update conversation route', () => { expect(response.status).toEqual(404); expect(response.body).toEqual({ - message: 'rule_id: "rule-1" not found', + message: 'conversation id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" not found', status_code: 404, }); }); test('catches error if search throws error', async () => { - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockImplementation( + clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockImplementation( async () => { throw new Error('Test error'); } @@ -91,44 +87,32 @@ describe('Update conversation route', () => { id: undefined, }, }); - const response = await server.inject(noIdRequest, requestContextMock.convertContext(context)); - expect(response.body).toEqual({ - message: ['either "id" or "rule_id" must be set'], - status_code: 400, - }); - }); - - test('allows query rule type', async () => { - const request = requestMock.create({ - method: 'put', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - body: { ...getUpdateConversationSchemaMock(), type: 'query' }, - }); - const result = await server.validate(request); - - expect(result.ok).toHaveBeenCalled(); + const response = await server.validate(noIdRequest); + expect(response.badRequest).toHaveBeenCalled(); }); - test('rejects unknown rule type', async () => { + test('rejects isDefault update', async () => { const request = requestMock.create({ method: 'put', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, - body: { ...getUpdateConversationSchemaMock(), type: 'unknown type' }, + body: { ...getUpdateConversationSchemaMock(), isDefault: false }, }); const result = await server.validate(request); expect(result.badRequest).toHaveBeenCalled(); }); - test('allows rule type of query and custom from and interval', async () => { + test('allows title, excludeFromLastConversationStorage, apiConfig, replacements and message', async () => { const request = requestMock.create({ method: 'put', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, body: { - from: 'now-7m', - interval: '5m', + title: 'test2', + excludeFromLastConversationStorage: true, ...getUpdateConversationSchemaMock(), - type: 'query', + apiConfig: { + defaultSystemPromptId: 'test', + }, }, }); const result = server.validate(request); @@ -136,19 +120,25 @@ describe('Update conversation route', () => { expect(result.ok).toHaveBeenCalled(); }); - test('disallows invalid "from" param on rule', async () => { + test('disallows invalid message "role" value', async () => { const request = requestMock.create({ method: 'put', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, body: { - from: 'now-3755555555555555.67s', - interval: '5m', ...getUpdateConversationSchemaMock(), - type: 'query', + messages: [ + { + role: 'invalid', + content: 'test', + timestamp: '2019-12-13T16:40:33.400Z', + }, + ], }, }); const result = server.validate(request); - expect(result.badRequest).toHaveBeenCalledWith('from: Failed to parse date-math expression'); + expect(result.badRequest).toHaveBeenCalledWith( + `messages.0.role: Invalid enum value. Expected 'system' | 'user' | 'assistant', received 'invalid'` + ); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts index 05a303db3c67e..717dcf131e823 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts @@ -121,27 +121,33 @@ describe('postActionsConnectorExecuteRoute', () => { it('returns the expected response when isEnabledKnowledgeBase=false', async () => { const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - const result = await handler( - mockContext, - { - ...mockRequest, - body: { - ...mockRequest.body, - isEnabledKnowledgeBase: false, - }, - }, - mockResponse - ); - - expect(result).toEqual({ - body: { - connector_id: 'mock-connector-id', - data: mockActionResponse, - status: 'ok', - }, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + const result = await handler( + mockContext, + { + ...mockRequest, + body: { + ...mockRequest.body, + isEnabledKnowledgeBase: false, + }, + }, + mockResponse + ); + + expect(result).toEqual({ + body: { + connector_id: 'mock-connector-id', + data: mockActionResponse, + status: 'ok', + }, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -152,18 +158,24 @@ describe('postActionsConnectorExecuteRoute', () => { it('returns the expected response when isEnabledKnowledgeBase=true', async () => { const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - const result = await handler(mockContext, mockRequest, mockResponse); - - expect(result).toEqual({ - body: { - connector_id: 'mock-connector-id', - data: mockActionResponse, - replacements: {}, - status: 'ok', - }, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + const result = await handler(mockContext, mockRequest, mockResponse); + + expect(result).toEqual({ + body: { + connector_id: 'mock-connector-id', + data: mockActionResponse, + replacements: {}, + status: 'ok', + }, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -179,14 +191,20 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - const result = await handler(mockContext, requestWithBadConnectorId, mockResponse); - - expect(result).toEqual({ - body: 'simulated error', - statusCode: 500, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + const result = await handler(mockContext, requestWithBadConnectorId, mockResponse); + + expect(result).toEqual({ + body: 'simulated error', + statusCode: 500, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -197,14 +215,20 @@ describe('postActionsConnectorExecuteRoute', () => { it('reports success events to telemetry - kb on, RAG alerts off', async () => { const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, mockRequest, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: false, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, mockRequest, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: false, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -226,14 +250,20 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, ragRequest, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: true, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, ragRequest, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: true, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -256,14 +286,20 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, req, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: false, - isEnabledRAGAlerts: true, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, req, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: true, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -282,14 +318,20 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, req, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { - isEnabledKnowledgeBase: false, - isEnabledRAGAlerts: false, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, req, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: false, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -305,15 +347,21 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, requestWithBadConnectorId, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { - errorMessage: 'simulated error', - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: false, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, requestWithBadConnectorId, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { + errorMessage: 'simulated error', + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: false, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -333,15 +381,21 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, badRequest, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { - errorMessage: 'simulated error', - isEnabledKnowledgeBase: true, - isEnabledRAGAlerts: true, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, badRequest, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { + errorMessage: 'simulated error', + isEnabledKnowledgeBase: true, + isEnabledRAGAlerts: true, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -365,15 +419,21 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, badRequest, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { - errorMessage: 'simulated error', - isEnabledKnowledgeBase: false, - isEnabledRAGAlerts: true, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, badRequest, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { + errorMessage: 'simulated error', + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: true, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( @@ -393,15 +453,21 @@ describe('postActionsConnectorExecuteRoute', () => { }; const mockRouter = { - post: jest.fn().mockImplementation(async (_, handler) => { - await handler(mockContext, badRequest, mockResponse); - - expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { - errorMessage: 'simulated error', - isEnabledKnowledgeBase: false, - isEnabledRAGAlerts: false, - }); - }), + versioned: { + post: jest.fn().mockImplementation(() => { + return { + addVersion: jest.fn().mockImplementation(async (_, handler) => { + await handler(mockContext, badRequest, mockResponse); + + expect(reportEvent).toHaveBeenCalledWith(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { + errorMessage: 'simulated error', + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: false, + }); + }), + }; + }), + }, }; await postActionsConnectorExecuteRoute( From 90b413dc900b62b212b46ea1b8470ec1c3221a3f Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Sun, 28 Jan 2024 22:04:47 -0800 Subject: [PATCH 053/141] type checks and ftr test --- .../create_conversation.ts | 2 +- .../server/lib/langchain/helpers.test.ts | 18 +++++++++--------- .../server/lib/langchain/helpers.ts | 4 ++-- .../langchain/llm/actions_client_llm.test.ts | 6 +++--- .../tests/actions/connector_types/bedrock.ts | 6 ++++++ 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index d9ab986934283..073df4a25945d 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -70,7 +70,7 @@ export const createConversation = async ({ conversation, }: CreateConversationParams): Promise => { const createdAt = new Date().toISOString(); - const body: CreateMessageSchema = transformToCreateScheme(createdAt, spaceId, user, conversation); + const body = transformToCreateScheme(createdAt, spaceId, user, conversation); const response = await esClient.create({ body, diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts index 20066afc68947..32f226f340e16 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts @@ -16,7 +16,7 @@ import { requestHasRequiredAnonymizationParams, } from './helpers'; import { langChainMessages } from '../../__mocks__/lang_chain_messages'; -import { RequestBody } from './types'; +import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('helpers', () => { describe('getLangChainMessage', () => { @@ -121,7 +121,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -135,7 +135,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -149,7 +149,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -163,7 +163,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -177,7 +177,7 @@ describe('helpers', () => { allowReplacement: [], replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -191,7 +191,7 @@ describe('helpers', () => { allowReplacement: ['b', 12345], // <-- non-string value replacements: { key: 'value' }, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -205,7 +205,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: {}, }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); @@ -219,7 +219,7 @@ describe('helpers', () => { allowReplacement: ['b', 'c'], replacements: { key: 76543 }, // <-- non-string value }, - } as unknown as KibanaRequest; + } as unknown as KibanaRequest; const result = requestHasRequiredAnonymizationParams(request); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts index 7850e2818e005..deb11bd9cf609 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts @@ -9,7 +9,7 @@ import { KibanaRequest } from '@kbn/core-http-server'; import type { Message } from '@kbn/elastic-assistant'; import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from 'langchain/schema'; -import { RequestBody } from './types'; +import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; export const getLangChainMessage = ( assistantMessage: Pick @@ -36,7 +36,7 @@ export const getMessageContentAndRole = (prompt: string): Pick + request: KibanaRequest ): boolean => { const { allow, allowReplacement, replacements } = request?.body ?? {}; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts index f2cffe7c9d41b..ad36eddfc0b6f 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts @@ -11,7 +11,7 @@ import { loggerMock } from '@kbn/logging-mocks'; import { ActionsClientLlm } from './actions_client_llm'; import { mockActionResponse } from '../../../__mocks__/action_result_data'; -import { RequestBody } from '../types'; +import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; const connectorId = 'mock-connector-id'; @@ -28,7 +28,7 @@ const mockActions = { })), } as unknown as ActionsPluginStart; -const mockRequest: KibanaRequest = { +const mockRequest: KibanaRequest = { params: { connectorId }, body: { params: { @@ -53,7 +53,7 @@ const mockRequest: KibanaRequest = { }, isEnabledKnowledgeBase: true, }, -} as KibanaRequest; +} as KibanaRequest; const prompt = 'Do you know my name?'; diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index 14ec27598a60f..edeb022fb734f 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -16,6 +16,10 @@ import { PassThrough } from 'stream'; import { EventStreamCodec } from '@smithy/eventstream-codec'; import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; import { TaskErrorSource } from '@kbn/task-manager-plugin/common'; +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; import { getUrlPrefix, ObjectRemover } from '../../../../../common/lib'; @@ -423,6 +427,8 @@ export default function bedrockTest({ getService }: FtrProviderContext) { supertest .post(`/internal/elastic_assistant/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .on('error', reject) .send({ params: { From d99f2747d03f7adeba572895d8f8a1788239b972 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Mon, 29 Jan 2024 21:11:52 -0800 Subject: [PATCH 054/141] fixed api tests and typechecks --- ...reate_resource_installation_helper.test.ts | 2 - .../create_resource_installation_helper.ts | 4 +- .../server/ai_assistant_service/index.test.ts | 128 ++++++++++++------ .../create_conversation.ts | 3 +- 4 files changed, 89 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts index 915ceb1032b2b..481211309123e 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.test.ts @@ -95,8 +95,6 @@ describe('createResourceInstallationHelper', () => { await retryUntil('init fns run', async () => logger.info.mock.calls.length === 3); expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); - // expect(logger.info).toHaveBeenNthCalledWith(2, 'test1_default'); - // expect(logger.info).toHaveBeenNthCalledWith(3, 'test2_default'); expect(await helper.getInitializedResources('test1')).toEqual({ result: true, }); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts index 8a00ecb029ea3..39e0e69a8fc49 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/create_resource_installation_helper.ts @@ -84,7 +84,7 @@ export function createResourceInstallationHelper( initPromise?: Promise, timeoutMs?: number ) => { - const key = `${namespace}`; + const key = namespace; // Use the new common initialization promise if specified if (initPromise) { commonInitPromise = initPromise; @@ -110,7 +110,7 @@ export function createResourceInstallationHelper( } }, getInitializedResources: async (namespace: string): Promise => { - const key = `${namespace}`; + const key = namespace; return ( initializedResources.has(key) ? initializedResources.get(key) diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index 4510d14e5cf1a..7fb45bd40f30e 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -121,8 +121,8 @@ describe('AI Assistant Service', () => { expect(assistantService.isInitialized()).toEqual(true); expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); - const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual( + const componentTemplate = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate.name).toEqual( '.kibana-elastic-ai-assistant-component-template-conversations' ); }); @@ -199,8 +199,6 @@ describe('AI Assistant Service', () => { expect(assistantService.isInitialized()).toEqual(false); - // Installing ILM policy failed so no calls to install context-specific resources - // should be made expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); @@ -231,10 +229,10 @@ describe('AI Assistant Service', () => { expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith( - `Installing component template .kibana-elastic-ai-assistant-component-template-conversations`, - 'Retrying common resource initialization', - `Retrying resource initialization for "default"` + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations` ); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying resource initialization for "default"`); }); test('should not retry initializing common resources if common resource initialization is in progress', async () => { @@ -259,8 +257,6 @@ describe('AI Assistant Service', () => { expect(assistantService.isInitialized()).toEqual(false); - // Installing ILM policy failed so no calls to install context-specific resources - // should be made expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); expect(clusterClient.indices.create).not.toHaveBeenCalled(); @@ -298,17 +294,19 @@ describe('AI Assistant Service', () => { expect(result[1]).not.toBe(null); expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations` ); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying resource initialization for "default"`); expect(logger.info).toHaveBeenCalledWith( - `Resource installation for "test" succeeded after retry` + `Resource installation for "default" succeeded after retry` ); expect(logger.info).toHaveBeenCalledWith( `Skipped retrying common resource initialization because it is already being retried.` ); }); - test('should retry initializing context specific resources if context specific resource initialization failed', async () => { + test('should retry initializing space specific resources if space specific resource initialization failed', async () => { clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ ...SimulateTemplateResponse, template: { @@ -316,7 +314,13 @@ describe('AI Assistant Service', () => { mappings: {}, }, })); - + clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); assistantService = new AIAssistantService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), @@ -346,12 +350,11 @@ describe('AI Assistant Service', () => { }); expect(result).not.toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); - expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` - ); + + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying resource initialization for "default"`); expect(logger.info).toHaveBeenCalledWith( - `Resource installation for "test" succeeded after retry` + `Resource installation for "default" succeeded after retry` ); }); @@ -365,6 +368,14 @@ describe('AI Assistant Service', () => { }, })); + clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + // this is the retry call that we'll artificially inflate the duration of clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => { await new Promise((r) => setTimeout(r, 1000)); @@ -413,20 +424,20 @@ describe('AI Assistant Service', () => { expect(result[0]).not.toBe(null); expect(result[1]).not.toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); // Should only log the retry once because the second call should // leverage the outcome of the first retry expect( logger.info.mock.calls.filter( // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + (calls: any[]) => calls[0] === `Retrying resource initialization for "default"` ).length ).toEqual(1); expect( logger.info.mock.calls.filter( // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => calls[0] === `Resource installation for "test" succeeded after retry` + (calls: any[]) => calls[0] === `Resource installation for "default" succeeded after retry` ).length ).toEqual(1); }); @@ -441,6 +452,14 @@ describe('AI Assistant Service', () => { }, })); + clusterClient.indices.simulateIndexTemplate.mockImplementation(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + assistantService = new AIAssistantService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), @@ -472,13 +491,13 @@ describe('AI Assistant Service', () => { createAIAssistantDatastreamClientWithDelay(2), ]); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); // Should only log the retry once because the second and third retries should be throttled expect( logger.info.mock.calls.filter( // eslint-disable-next-line @typescript-eslint/no-explicit-any - (calls: any[]) => calls[0] === `Retrying resource initialization for context "test"` + (calls: any[]) => calls[0] === `Retrying resource initialization for "default"` ).length ).toEqual(1); }); @@ -497,12 +516,10 @@ describe('AI Assistant Service', () => { taskManager: taskManagerMock.createSetup(), }); - await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0, 1); expect(assistantService.isInitialized()).toEqual(false); - // Installing ILM policy failed so no calls to install context-specific resources - // should be made expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); @@ -510,7 +527,7 @@ describe('AI Assistant Service', () => { const result = await assistantService.createAIAssistantDatastreamClient({ logger, - spaceId: 'default', + spaceId: 'test', currentUser: mockUser1, }); @@ -523,12 +540,19 @@ describe('AI Assistant Service', () => { expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations` + ); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying resource initialization for "test"`); + + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize resources for test` + ); + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize resources for test` ); expect(logger.warn).toHaveBeenCalledWith( - expect.stringMatching( - /There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation\. fail \d+; Error after retry: Failure during installation\. fail \d+/ - ) + `There was an error in the framework installing spaceId-level resources and creating concrete indices for spaceId "test" - Retry failed with errors: Failure during installation of create or update .kibana-elastic-ai-assistant-component-template-conversations component template. fail 1` ); }); @@ -556,7 +580,7 @@ describe('AI Assistant Service', () => { const result = await assistantService.createAIAssistantDatastreamClient({ logger, - spaceId: 'default', + spaceId: 'test', currentUser: mockUser1, }); @@ -569,10 +593,18 @@ describe('AI Assistant Service', () => { expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "test"` + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations` + ); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying resource initialization for "test"`); + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize resources for test` + ); + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize resources for test` ); expect(logger.warn).toHaveBeenCalledWith( - `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Retry failed with error: Failure during installation. fail` + `There was an error in the framework installing spaceId-level resources and creating concrete indices for spaceId "test" - Retry failed with errors: Failure during installation of create or update .kibana-elastic-ai-assistant-component-template-conversations component template. fail` ); }); @@ -603,18 +635,25 @@ describe('AI Assistant Service', () => { const result = await assistantService.createAIAssistantDatastreamClient({ logger, - spaceId: 'default', + spaceId: 'test', currentUser: mockUser1, }); expect(AIAssistantConversationsDataClient).not.toHaveBeenCalled(); expect(result).toBe(null); - expect(logger.info).not.toHaveBeenCalledWith(`Retrying common resource initialization`); + expect(logger.info).toHaveBeenCalledWith(`Retrying common resource initialization`); expect(logger.info).toHaveBeenCalledWith( - `Retrying resource initialization for context "default"` + `Installing component template .kibana-elastic-ai-assistant-component-template-conversations` + ); + + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize resources for test` ); expect(logger.warn).toHaveBeenCalledWith( - `There was an error in the framework installing namespace-level resources and creating concrete indices for context "test" - Original error: Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping; Error after retry: Failure during installation. fail index template` + `Common resources were not initialized, cannot initialize resources for test` + ); + expect(logger.warn).toHaveBeenCalledWith( + `There was an error in the framework installing spaceId-level resources and creating concrete indices for spaceId \"test\" - Retry failed with errors: Failure during installation of create or update .kibana-elastic-ai-assistant-index-template-conversations index template. No mappings would be generated for .kibana-elastic-ai-assistant-index-template-conversations, possibly due to failed/misconfigured bootstrapping` ); }); }); @@ -720,17 +759,14 @@ describe('AI Assistant Service', () => { }); test('should retry creating concrete index for transient ES errors', async () => { - clusterClient.indices.getDataStream.mockImplementationOnce(async () => ({ + clusterClient.indices.getDataStream.mockImplementation(async () => ({ data_streams: [], })); clusterClient.indices.createDataStream .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) .mockResolvedValue({ acknowledged: true }); - clusterClient.indices.create - .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) - .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) - .mockResolvedValue({ index: 'index', shards_acknowledged: true, acknowledged: true }); + const assistantService = new AIAssistantService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), @@ -744,6 +780,12 @@ describe('AI Assistant Service', () => { async () => assistantService.isInitialized() === true ); + await assistantService.createAIAssistantDatastreamClient({ + logger, + spaceId: 'default', + currentUser: mockUser1, + }); + await retryUntil( 'space resources initialized', async () => (await getSpaceResourcesInitialized(assistantService)) === true diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 073df4a25945d..38853f37d4ea5 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -11,6 +11,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { ConversationCreateProps, ConversationResponse, + Message, Replacement, UUID, } from '../schemas/conversations/common_attributes.gen'; @@ -153,7 +154,7 @@ function transform(conversationSchema: CreateMessageSchema): ConversationRespons presentation: message.presentation, reader: message.reader, replacements: message.replacements as Replacement[], - role: message.role, + role: message.role as Message['role'], traceData: { traceId: message.trace_data?.trace_id, transactionId: message.trace_data?.transaction_id, From a1fe21af47c3a120edb7b2b7859f3f24ac48f026 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 30 Jan 2024 21:49:16 -0800 Subject: [PATCH 055/141] fixed first batch of comments --- .../api/conversations/conversations.ts | 8 +- .../use_bulk_actions_conversations.ts | 83 ++++++++-------- .../impl/assistant/assistant_header/index.tsx | 4 +- .../conversation_selector/index.tsx | 16 ++-- .../impl/assistant/helpers.ts | 27 +++++- .../impl/assistant/index.tsx | 61 +++++------- .../conversation_multi_selector.tsx | 1 - .../assistant/settings/assistant_settings.tsx | 4 +- .../settings/assistant_settings_button.tsx | 2 +- .../assistant/use_conversation/index.test.tsx | 18 ++-- .../impl/assistant/use_conversation/index.tsx | 20 ++-- .../impl/assistant_context/types.tsx | 3 +- .../connectorland/connector_setup/index.tsx | 4 +- .../packages/kbn-elastic-assistant/index.ts | 2 + .../conversation_configuration_type.ts | 2 +- .../append_conversation_messages.ts | 40 +++++--- .../create_conversation.ts | 25 +++-- .../get_conversation.ts | 22 +++-- .../server/conversations_data_client/index.ts | 32 ++++--- .../conversations_data_client/transforms.ts | 95 ++++++++++--------- .../update_conversation.ts | 34 ++++--- .../append_conversation_messages_route.ts | 8 +- .../routes/conversations/create_route.ts | 3 +- .../routes/conversations/delete_route.ts | 6 +- .../conversations/common_attributes.gen.ts | 25 ++--- .../common_attributes.schema.yaml | 29 +++--- .../public/assistant/provider.tsx | 25 ++++- .../use_conversation_store/index.tsx | 65 +------------ 28 files changed, 344 insertions(+), 320 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts index 0950b16760456..cef370368e1d6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts @@ -64,7 +64,7 @@ export interface PostConversationParams { * * @returns {Promise} */ -export const createConversationApi = async ({ +export const createConversation = async ({ http, conversation, signal, @@ -102,7 +102,7 @@ export interface DeleteConversationResponse { * * @returns {Promise} */ -export const deleteConversationApi = async ({ +export const deleteConversation = async ({ http, id, signal, @@ -146,7 +146,7 @@ export interface PutConversationMessageParams { * * @returns {Promise} */ -export const updateConversationApi = async ({ +export const updateConversation = async ({ http, title, conversationId, @@ -188,7 +188,7 @@ export const updateConversationApi = async ({ * * @returns {Promise} */ -export const appendConversationMessagesApi = async ({ +export const appendConversationMessages = async ({ http, conversationId, messages, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 97e3d0a634f0a..089872a4dc220 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -69,54 +69,57 @@ export interface ConversationsBulkActions { }; } +const transformCreateActions = ( + createActions: Record, + conversationIdsToDelete?: string[] +) => + Object.keys(createActions).reduce((conversationsToCreate: Conversation[], conversationId) => { + if (createActions && !conversationIdsToDelete?.includes(conversationId)) { + conversationsToCreate.push(createActions[conversationId]); + } + return conversationsToCreate; + }, []); + +const transformUpdateActions = ( + updateActions: Record, + conversationIdsToDelete?: string[] +) => + Object.keys(updateActions).reduce( + (conversationsToUpdate: ConversationUpdateParams[], conversationId) => { + if (updateActions && !conversationIdsToDelete?.includes(conversationId)) { + conversationsToUpdate.push({ + id: conversationId, + ...updateActions[conversationId], + }); + } + return conversationsToUpdate; + }, + [] + ); + export const bulkChangeConversations = ( http: HttpSetup, conversationsActions: ConversationsBulkActions ) => { - const conversationIdsToDelete = conversationsActions.delete?.ids.filter( - (cId) => !(conversationsActions.create ?? {})[cId] && !(conversationsActions.update ?? {})[cId] - ); + // transform conversations disctionary to array of Conversations to create + // filter marked as deleted + const conversationsToCreate = conversationsActions.create + ? transformCreateActions(conversationsActions.create, conversationsActions.delete?.ids) + : undefined; + + // transform conversations disctionary to array of Conversations to update + // filter marked as deleted + const conversationsToUpdate = conversationsActions.update + ? transformUpdateActions(conversationsActions.update, conversationsActions.delete?.ids) + : undefined; + return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { method: 'POST', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, body: JSON.stringify({ - update: conversationsActions.update - ? Object.keys(conversationsActions.update).reduce( - (conversationsToUpdate: ConversationUpdateParams[], conversationId) => { - if ( - conversationsActions.update && - !conversationsActions.delete?.ids.includes(conversationId) - ) { - conversationsToUpdate.push({ - id: conversationId, - ...conversationsActions.update[conversationId], - }); - } - return conversationsToUpdate; - }, - [] - ) - : undefined, - create: conversationsActions.create - ? Object.keys(conversationsActions.create).reduce( - (conversationsToCreate: Conversation[], conversationId: string) => { - if ( - conversationsActions.create && - !conversationsActions.delete?.ids.includes(conversationId) - ) { - conversationsToCreate.push(conversationsActions.create[conversationId]); - } - return conversationsToCreate; - }, - [] - ) - : undefined, - delete: - conversationIdsToDelete && conversationIdsToDelete.length > 0 - ? { - ids: conversationIdsToDelete, - } - : undefined, + update: conversationsToUpdate, + create: conversationsToCreate, + delete: conversationsActions.delete?.ids, }), }); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index 625091f6b5b77..35599e0ccc8c6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -31,7 +31,7 @@ interface OwnProps { docLinks: Omit; isDisabled: boolean; isSettingsModalVisible: boolean; - onConversationSelected: (cId: string) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; onConversationDeleted: (conversationId: string) => void; onToggleShowAnonymizedValues: (e: EuiSwitchEvent) => void; setIsSettingsModalVisible: React.Dispatch>; @@ -90,7 +90,7 @@ export const AssistantHeader: React.FC = ({ selectedConversation={currentConversation} onChange={(updatedConversation) => { setCurrentConversation(updatedConversation); - onConversationSelected(updatedConversation.id); + onConversationSelected({ cId: updatedConversation.id }); }} title={title} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index e193fbd62a9d3..e7ee7656aec05 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -33,7 +33,7 @@ interface Props { defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; selectedConversationId: string | undefined; - onConversationSelected: (conversationId: string, title?: string) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; onConversationDeleted: (conversationId: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; @@ -114,7 +114,7 @@ export const ConversationSelector: React.FC = React.memo( }; cId = (await createConversation(newConversation))?.id; } - onConversationSelected(cId ?? DEFAULT_CONVERSATION_TITLE); + onConversationSelected({ cId: cId ?? DEFAULT_CONVERSATION_TITLE }); }, [ allSystemPrompts, @@ -131,7 +131,10 @@ export const ConversationSelector: React.FC = React.memo( onConversationDeleted(cId); if (selectedConversationId === cId) { const prevConversationId = getPreviousConversationId(conversationIds, cId); - onConversationSelected(prevConversationId, conversations[prevConversationId].title); + onConversationSelected({ + cId: prevConversationId, + cTitle: conversations[prevConversationId].title, + }); } }, [ @@ -148,7 +151,8 @@ export const ConversationSelector: React.FC = React.memo( if (newOptions.length === 0 || !newOptions?.[0].id) { setSelectedOptions([]); } else if (conversationOptions.findIndex((o) => o.id === newOptions?.[0].id) !== -1) { - await onConversationSelected(newOptions?.[0].id, newOptions?.[0].label); + const { id, label } = newOptions?.[0]; + await onConversationSelected({ cId: id, cTitle: label }); } }, [conversationOptions, onConversationSelected] @@ -156,11 +160,11 @@ export const ConversationSelector: React.FC = React.memo( const onLeftArrowClick = useCallback(() => { const prevId = getPreviousConversationId(conversationIds, selectedConversationId); - onConversationSelected(prevId, conversations[prevId].title); + onConversationSelected({ cId: prevId, cTitle: conversations[prevId].title }); }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); const onRightArrowClick = useCallback(() => { const nextId = getNextConversationId(conversationIds, selectedConversationId); - onConversationSelected(nextId, conversations[nextId].title); + onConversationSelected({ cId: nextId, cTitle: conversations[nextId].title }); }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); // Register keyboard listener for quick conversation switching diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index f7ea3f52c8826..9db98dda66b65 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -6,7 +6,8 @@ */ import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public'; -import { FetchConnectorExecuteResponse } from './api'; +import { merge } from 'lodash/fp'; +import { FetchConnectorExecuteResponse, FetchConversationsResponse } from './api'; import { Conversation } from '../..'; import type { Message } from '../assistant_context/types'; import { enterpriseMessaging, WELCOME_CONVERSATION } from './use_conversation/sample_conversations'; @@ -34,6 +35,30 @@ export const getMessageFromRawResponse = (rawResponse: FetchConnectorExecuteResp } }; +export const mergeBaseWithPersistedConversations = ( + baseConversations: Record, + conversationsData: FetchConversationsResponse +): Record => { + const userConversations = (conversationsData?.data ?? []).reduce>( + (transformed, conversation) => { + transformed[conversation.id] = conversation; + return transformed; + }, + {} + ); + return merge( + userConversations, + Object.keys(baseConversations) + .filter( + (baseId) => (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined + ) + .reduce>((transformed, conversation) => { + transformed[conversation] = baseConversations[conversation]; + return transformed; + }, {}) + ); +}; + export const getBlockBotConversation = ( conversation: Conversation, isAssistantEnabled: boolean diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index dadf5387522bc..a994afe227816 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -31,13 +31,16 @@ import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { ActionConnectorProps } from '@kbn/triggers-actions-ui-plugin/public/types'; -import { merge } from 'lodash'; import { useChatSend } from './chat_send/use_chat_send'; import { ChatSend } from './chat_send'; import { BlockBotCallToAction } from './block_bot/cta'; import { AssistantHeader } from './assistant_header'; import { WELCOME_CONVERSATION_TITLE } from './use_conversation/translations'; -import { getDefaultConnector, getBlockBotConversation } from './helpers'; +import { + getDefaultConnector, + getBlockBotConversation, + mergeBaseWithPersistedConversations, +} from './helpers'; import { useAssistantContext } from '../assistant_context'; import { ContextPills } from './context_pills'; @@ -108,26 +111,8 @@ const AssistantComponent: React.FC = ({ ); const onFetchedConversations = useCallback( - (conversationsData: FetchConversationsResponse): Record => { - const userConversations = (conversationsData?.data ?? []).reduce< - Record - >((transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, {}); - return merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ); - }, + (conversationsData: FetchConversationsResponse): Record => + mergeBaseWithPersistedConversations(baseConversations, conversationsData), [baseConversations] ); const { @@ -312,7 +297,7 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - async (cId: string, cTitle?: string) => { + async ({ cId, cTitle }: { cId: string; cTitle?: string }) => { if (conversations[cId] === undefined && cId) { const updatedConv = await refetchResults(); if (updatedConv) { @@ -559,6 +544,21 @@ const AssistantComponent: React.FC = ({ [assistantTelemetry, selectedConversationId] ); + const refetchConversationsState = useCallback(async () => { + const refetchedConversations = await refetchResults(); + if (refetchedConversations && refetchedConversations[selectedConversationId]) { + setCurrentConversation(refetchedConversations[selectedConversationId]); + } else if (refetchedConversations) { + const createdSelectedConversation = Object.values(refetchedConversations).find( + (c) => c.title === selectedConversationId + ); + if (createdSelectedConversation) { + setCurrentConversation(createdSelectedConversation); + setSelectedConversationId(createdSelectedConversation.id); + } + } + }, [refetchResults, selectedConversationId]); + return getWrapper( <> = ({ title={title} conversations={conversations} onConversationDeleted={handleOnConversationDeleted} - refetchConversationsState={async () => { - const refetchedConversations = await refetchResults(); - if (refetchedConversations && refetchedConversations[selectedConversationId]) { - setCurrentConversation(refetchedConversations[selectedConversationId]); - } else if (refetchedConversations) { - const createdSelectedConversation = Object.values(refetchedConversations).find( - (c) => c.title === selectedConversationId - ); - if (createdSelectedConversation) { - setCurrentConversation(createdSelectedConversation); - setSelectedConversationId(createdSelectedConversation.id); - } - } - }} + refetchConversationsState={refetchConversationsState} /> )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx index d13c8146621cf..fe2318622114f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/conversation_multi_selector/conversation_multi_selector.tsx @@ -33,7 +33,6 @@ export const ConversationMultiSelector: React.FC = React.memo( const options = useMemo( () => conversations.map((conversation) => ({ - // id: conversation.id label: conversation.title ?? '', 'data-test-subj': TEST_IDS.CONVERSATIONS_MULTISELECTOR_OPTION(conversation.id), })), diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index 7f7ed54f5301b..4e386557db633 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -65,7 +65,7 @@ interface Props { ) => void; onSave: () => Promise; selectedConversation: Conversation; - onConversationSelected: (cId: string) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; conversations: Record; } @@ -150,7 +150,7 @@ export const AssistantSettings: React.FC = React.memo( conversationSettings[defaultSelectedConversation.id] == null; const newSelectedConversationId: string | undefined = Object.keys(conversationSettings)[0]; if (isSelectedConversationDeleted && newSelectedConversationId != null) { - onConversationSelected(newSelectedConversationId); + onConversationSelected({ cId: newSelectedConversationId }); } await saveSettings(); await onSave(); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index 4d294c1b123fb..acec91064d72a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -20,7 +20,7 @@ interface Props { isSettingsModalVisible: boolean; selectedConversation: Conversation; setIsSettingsModalVisible: React.Dispatch>; - onConversationSelected: (cId: string) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; isDisabled?: boolean; conversations: Record; refetchConversationsState: () => Promise; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index c0b581cf7bd22..91648c721bf08 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -14,11 +14,11 @@ import { ConversationRole } from '../../assistant_context/types'; import { httpServiceMock } from '@kbn/core/public/mocks'; import { WELCOME_CONVERSATION } from './sample_conversations'; import { - appendConversationMessagesApi as _appendConversationMessagesApi, - deleteConversationApi, + appendConversationMessages as _appendConversationMessagesApi, + deleteConversation, getConversationById as _getConversationById, - updateConversationApi, - createConversationApi as _createConversationApi, + updateConversation, + createConversation as _createConversationApi, } from '../api/conversations'; jest.mock('../api/conversations'); @@ -42,7 +42,7 @@ const mockConvo = { const appendConversationMessagesApi = _appendConversationMessagesApi as jest.Mock; const getConversationById = _getConversationById as jest.Mock; -const createConversationApi = _createConversationApi as jest.Mock; +const createConversation = _createConversationApi as jest.Mock; describe('useConversation', () => { let httpMock: ReturnType; @@ -114,7 +114,7 @@ describe('useConversation', () => { ), }); await waitForNextUpdate(); - createConversationApi.mockResolvedValue(mockConvo); + createConversation.mockResolvedValue(mockConvo); const createResult = await result.current.createConversation({ id: mockConvo.id, @@ -138,7 +138,7 @@ describe('useConversation', () => { await result.current.deleteConversation('new-convo'); - expect(deleteConversationApi).toHaveBeenCalledWith({ + expect(deleteConversation).toHaveBeenCalledWith({ http: httpMock, id: 'new-convo', }); @@ -159,7 +159,7 @@ describe('useConversation', () => { apiConfig: mockConvo.apiConfig, }); - expect(createConversationApi).toHaveBeenCalledWith({ + expect(createConversation).toHaveBeenCalledWith({ http: httpMock, conversation: { ...WELCOME_CONVERSATION, apiConfig: mockConvo.apiConfig, id: '' }, }); @@ -186,7 +186,7 @@ describe('useConversation', () => { }, }); - expect(updateConversationApi).toHaveBeenCalledWith({ + expect(updateConversation).toHaveBeenCalledWith({ http: httpMock, conversationId: welcomeConvo.id, replacements: { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 83dcad0f51441..c26274d5ca812 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -13,11 +13,11 @@ import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; import { getDefaultSystemPrompt } from './helpers'; import { - appendConversationMessagesApi, - createConversationApi, - deleteConversationApi, + appendConversationMessages, + createConversation as createConversationApi, + deleteConversation as deleteConversationApi, getConversationById, - updateConversationApi, + updateConversation, } from '../api/conversations'; import { WELCOME_CONVERSATION } from './sample_conversations'; @@ -105,7 +105,7 @@ export const useConversation = (): UseConversation => { } if (prevConversation != null) { messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); - await updateConversationApi({ + await updateConversation({ http, conversationId, messages, @@ -130,7 +130,7 @@ export const useConversation = (): UseConversation => { const message = messages[messages.length - 1]; const updatedMessages = message ? [{ ...message, content }] : []; - await appendConversationMessagesApi({ + await appendConversationMessages({ http, conversationId, messages: updatedMessages, @@ -152,7 +152,7 @@ export const useConversation = (): UseConversation => { isEnabledRAGAlerts, }); - const res = await appendConversationMessagesApi({ + const res = await appendConversationMessages({ http, conversationId, messages: [message], @@ -181,7 +181,7 @@ export const useConversation = (): UseConversation => { ...replacements, }; - await updateConversationApi({ + await updateConversation({ http, conversationId, replacements: allReplacements, @@ -204,7 +204,7 @@ export const useConversation = (): UseConversation => { conversation: prevConversation, })?.id; - await updateConversationApi({ + await updateConversation({ http, conversationId, apiConfig: { @@ -286,7 +286,7 @@ export const useConversation = (): UseConversation => { }, }); } else { - return updateConversationApi({ + return updateConversation({ http, conversationId: conversation.id, apiConfig, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 568b659bb1ebf..0a17f86010405 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -65,7 +65,8 @@ export interface Conversation { id: string; title: string; messages: Message[]; - updatedAt?: string; + updatedAt?: Date; + createdAt?: Date; replacements?: Record; isDefault?: boolean; excludeFromLastConversationStorage?: boolean; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 3b59353ea8227..82fe0a2e98fb7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -39,7 +39,7 @@ const SkipEuiText = styled(EuiText)` export interface ConnectorSetupProps { conversation?: Conversation; onSetupComplete?: () => void; - onConversationUpdate: (cId: string, cTitle?: string) => Promise; + onConversationUpdate: ({ cId, cTitle }: { cId: string; cTitle?: string }) => Promise; } export const useConnectorSetup = ({ @@ -190,7 +190,7 @@ export const useConnectorSetup = ({ }); if (!isHttpFetchError(updatedConversation)) { - onConversationUpdate(updatedConversation.id, updatedConversation.title); + onConversationUpdate({ cId: updatedConversation.id, cTitle: updatedConversation.title }); refetchConnectors?.(); setIsConnectorModalVisible(false); diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 8c79f87161408..d02836f04a38d 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -144,3 +144,5 @@ export type { PostKnowledgeBaseResponse } from './impl/assistant/api'; export { useFetchCurrentUserConversations } from './impl/assistant/api/conversations/use_fetch_current_user_conversations'; export * from './impl/assistant/api/conversations/use_bulk_actions_conversations'; export { getConversationById } from './impl/assistant/api/conversations/conversations'; + +export { mergeBaseWithPersistedConversations } from './impl/assistant/helpers'; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts index 29e499cd594e2..d246845ee5f1d 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts @@ -75,7 +75,7 @@ export const conversationsFieldMap: FieldMap = { required: false, }, 'messages.reader': { - type: 'keyword', + type: 'object', array: false, required: false, }, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index dd51f24c23802..d2fbb51303f9b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -14,14 +14,23 @@ import { } from '../schemas/conversations/common_attributes.gen'; import { getConversation } from './get_conversation'; -export const appendConversationMessages = async ( - esClient: ElasticsearchClient, - logger: Logger, - conversationIndex: string, - userId: UUID, - existingConversation: ConversationResponse, - messages: Message[] -): Promise => { +export interface AppendConversationMessagesParams { + esClient: ElasticsearchClient; + logger: Logger; + conversationIndex: string; + user: { id?: UUID; name?: string }; + existingConversation: ConversationResponse; + messages: Message[]; +} + +export const appendConversationMessages = async ({ + esClient, + logger, + conversationIndex, + user, + existingConversation, + messages, +}: AppendConversationMessagesParams): Promise => { const updatedAt = new Date().toISOString(); const params = transformToUpdateScheme(updatedAt, [ @@ -71,18 +80,21 @@ export const appendConversationMessages = async ( ); return null; } + + const updatedConversation = await getConversation({ + esClient, + conversationIndex, + id: existingConversation.id, + logger, + user, + }); + return updatedConversation; } catch (err) { logger.warn( `Error appending conversation messages: ${err} for conversation by ID: ${existingConversation.id}` ); throw err; } - const updatedConversation = await getConversation( - esClient, - conversationIndex, - existingConversation.id ?? '' - ); - return updatedConversation; }; export const transformToUpdateScheme = (updatedAt: string, messages: Message[]) => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 38853f37d4ea5..56e8d0390a7f8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -11,7 +11,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { ConversationCreateProps, ConversationResponse, - Message, + Reader, Replacement, UUID, } from '../schemas/conversations/common_attributes.gen'; @@ -24,8 +24,8 @@ export interface CreateMessageSchema { messages?: Array<{ '@timestamp': string; content: string; - reader?: string | undefined; - replacements?: unknown; + reader?: Reader; + replacements?: Replacement; role: 'user' | 'assistant' | 'system'; is_error?: boolean; presentation?: { @@ -46,8 +46,8 @@ export interface CreateMessageSchema { }; is_default?: boolean; exclude_from_last_conversation_storage?: boolean; - replacements?: unknown; - user?: { + replacements?: Replacement; + user: { id?: string; name?: string; }; @@ -81,8 +81,8 @@ export const createConversation = async ({ }); return { - id: response._id, ...transform(body), + id: response._id, }; }; @@ -132,8 +132,8 @@ export const transformToCreateScheme = ( }; }; -function transform(conversationSchema: CreateMessageSchema): ConversationResponse { - const response: ConversationResponse = { +function transform(conversationSchema: CreateMessageSchema): Omit { + return { timestamp: conversationSchema['@timestamp'], createdAt: conversationSchema.created_at, user: conversationSchema.user, @@ -143,7 +143,7 @@ function transform(conversationSchema: CreateMessageSchema): ConversationRespons connectorTypeTitle: conversationSchema.api_config?.connector_type_title, defaultSystemPromptId: conversationSchema.api_config?.default_system_prompt_id, model: conversationSchema.api_config?.model, - provider: conversationSchema.api_config?.provider, + provider: conversationSchema.api_config?.provider as 'OpenAI' | 'Azure OpenAI' | undefined, }, excludeFromLastConversationStorage: conversationSchema.exclude_from_last_conversation_storage, isDefault: conversationSchema.is_default, @@ -153,16 +153,15 @@ function transform(conversationSchema: CreateMessageSchema): ConversationRespons isError: message.is_error, presentation: message.presentation, reader: message.reader, - replacements: message.replacements as Replacement[], - role: message.role as Message['role'], + replacements: message.replacements as Replacement, + role: message.role, traceData: { traceId: message.trace_data?.trace_id, transactionId: message.trace_data?.transaction_id, }, })), updatedAt: conversationSchema.updated_at, - replacements: conversationSchema.replacements as Replacement[], + replacements: conversationSchema.replacements as Replacement, namespace: conversationSchema.namespace, }; - return response; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts index 7b1606d0d180f..3cc0244b5419c 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -5,16 +5,24 @@ * 2.0. */ -import { ElasticsearchClient } from '@kbn/core/server'; -import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { ConversationResponse, UUID } from '../schemas/conversations/common_attributes.gen'; import { SearchEsConversationSchema } from './types'; import { transformESToConversations } from './transforms'; -export const getConversation = async ( - esClient: ElasticsearchClient, - conversationIndex: string, - id: string -): Promise => { +export interface GetConversationParams { + esClient: ElasticsearchClient; + logger: Logger; + conversationIndex: string; + id: string; + user: { id?: UUID; name?: string }; +} + +export const getConversation = async ({ + esClient, + conversationIndex, + id, +}: GetConversationParams): Promise => { const response = await esClient.search({ body: { query: { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index 29d1d4b2d94c4..d032f24935e9e 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -115,7 +115,13 @@ export class AIAssistantConversationsDataClient { public getConversation = async (id: string): Promise => { const esClient = await this.options.elasticsearchClientPromise; - return getConversation(esClient, this.indexTemplateAndPattern.alias, id); + return getConversation({ + esClient, + logger: this.options.logger, + conversationIndex: this.indexTemplateAndPattern.alias, + id, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + }); }; /** @@ -126,19 +132,19 @@ export class AIAssistantConversationsDataClient { * @returns The conversation updated */ public appendConversationMessages = async ( - conversation: ConversationResponse, + existingConversation: ConversationResponse, messages: Message[] ): Promise => { const { currentUser } = this; const esClient = await this.options.elasticsearchClientPromise; - return appendConversationMessages( + return appendConversationMessages({ esClient, - this.options.logger, - this.indexTemplateAndPattern.alias, - currentUser?.profile_uid ?? '', - conversation, - messages - ); + logger: this.options.logger, + conversationIndex: this.indexTemplateAndPattern.alias, + user: { id: currentUser?.profile_uid, name: currentUser?.username }, + existingConversation, + messages, + }); }; public findConversations = async ({ @@ -199,6 +205,7 @@ export class AIAssistantConversationsDataClient { * See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html} * for more information around optimistic concurrency control. * @param options + * @param options.conversationUpdateProps * @param options.id id of the conversation to replace the conversation container data with. * @param options.title The new tilet, or "undefined" if this should not be updated. * @param options.messages The new messages, or "undefined" if this should not be updated. @@ -207,7 +214,7 @@ export class AIAssistantConversationsDataClient { */ public updateConversation = async ( existingConversation: ConversationResponse, - updatedProps: ConversationUpdateProps, + conversationUpdateProps: ConversationUpdateProps, isPatch?: boolean ): Promise => { const esClient = await this.options.elasticsearchClientPromise; @@ -216,8 +223,9 @@ export class AIAssistantConversationsDataClient { logger: this.options.logger, conversationIndex: this.indexTemplateAndPattern.alias, existingConversation, - conversation: updatedProps, + conversationUpdateProps, isPatch, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, }); }; @@ -227,7 +235,7 @@ export class AIAssistantConversationsDataClient { * @param options.id The id of the conversation to delete * @returns The conversation deleted if found, otherwise null */ - public deleteConversation = async (id: string): Promise => { + public deleteConversation = async (id: string) => { const esClient = await this.options.elasticsearchClientPromise; await deleteConversation({ esClient, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index d31fd3d72a22c..9e2eab800f9ec 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -12,53 +12,56 @@ import { ConversationResponse, Replacement } from '../schemas/conversations/comm export const transformESToConversations = ( response: estypes.SearchResponse ): ConversationResponse[] => { - return response.hits.hits.map((hit) => { - const conversationSchema = hit._source; - const conversation: ConversationResponse = { - timestamp: conversationSchema?.['@timestamp'], - createdAt: conversationSchema?.created_at, - user: { - id: conversationSchema?.user?.id, - name: conversationSchema?.user?.name, - }, - title: conversationSchema?.title, - apiConfig: { - connectorId: conversationSchema?.api_config?.connector_id, - connectorTypeTitle: conversationSchema?.api_config?.connector_type_title, - defaultSystemPromptId: conversationSchema?.api_config?.default_system_prompt_id, - model: conversationSchema?.api_config?.model, - provider: conversationSchema?.api_config?.provider, - }, - excludeFromLastConversationStorage: - conversationSchema?.exclude_from_last_conversation_storage, - isDefault: conversationSchema?.is_default, - messages: - // eslint-disable-next-line @typescript-eslint/no-explicit-any - conversationSchema?.messages?.map((message: Record) => ({ - timestamp: message['@timestamp'], - content: message.content, - ...(message.is_error ? { isError: message.is_error } : {}), - ...(message.presentation ? { presentation: message.presentation } : {}), - ...(message.reader ? { reader: message.reader } : {}), - ...(message.replacements ? { replacements: message.replacements as Replacement[] } : {}), - role: message.role, - ...(message.trace_data - ? { - traceData: { - traceId: message.trace_data?.trace_id, - transactionId: message.trace_data?.transaction_id, - }, - } - : {}), - })) ?? [], - updatedAt: conversationSchema?.updated_at, - replacements: conversationSchema?.replacements as Replacement[], - namespace: conversationSchema?.namespace, - id: hit._id, - }; + return response.hits.hits + .filter((hit) => hit._source !== undefined) + .map((hit) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const conversationSchema = hit._source!; + const conversation: ConversationResponse = { + timestamp: conversationSchema['@timestamp'], + createdAt: conversationSchema.created_at, + user: { + id: conversationSchema.user?.id, + name: conversationSchema.user?.name, + }, + title: conversationSchema.title, + apiConfig: { + connectorId: conversationSchema.api_config?.connector_id, + connectorTypeTitle: conversationSchema.api_config?.connector_type_title, + defaultSystemPromptId: conversationSchema.api_config?.default_system_prompt_id, + model: conversationSchema.api_config?.model, + provider: conversationSchema.api_config?.provider, + }, + excludeFromLastConversationStorage: + conversationSchema.exclude_from_last_conversation_storage, + isDefault: conversationSchema.is_default, + messages: + // eslint-disable-next-line @typescript-eslint/no-explicit-any + conversationSchema.messages?.map((message: Record) => ({ + timestamp: message['@timestamp'], + content: message.content, + ...(message.is_error ? { isError: message.is_error } : {}), + ...(message.presentation ? { presentation: message.presentation } : {}), + ...(message.reader ? { reader: message.reader } : {}), + ...(message.replacements ? { replacements: message.replacements as Replacement } : {}), + role: message.role, + ...(message.trace_data + ? { + traceData: { + traceId: message.trace_data?.trace_id, + transactionId: message.trace_data?.transaction_id, + }, + } + : {}), + })) ?? [], + updatedAt: conversationSchema.updated_at, + replacements: conversationSchema.replacements as Replacement, + namespace: conversationSchema.namespace, + id: hit._id, + }; - return conversation; - }); + return conversation; + }); }; export const encodeHitVersion = (hit: T): string | undefined => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index e773f9e9779ac..a4e1f7744a65c 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -8,7 +8,10 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { ConversationResponse, + Replacement, + Reader, ConversationUpdateProps, + UUID, } from '../schemas/conversations/common_attributes.gen'; import { getConversation } from './get_conversation'; @@ -18,8 +21,8 @@ export interface UpdateConversationSchema { messages?: Array<{ '@timestamp': string; content: string; - reader?: string | undefined; - replacements?: unknown; + reader?: Reader; + replacements?: Replacement; role: 'user' | 'assistant' | 'system'; is_error?: boolean; presentation?: { @@ -39,16 +42,17 @@ export interface UpdateConversationSchema { model?: string; }; exclude_from_last_conversation_storage?: boolean; - replacements?: unknown; + replacements?: Replacement; updated_at?: string; } export interface UpdateConversationParams { esClient: ElasticsearchClient; logger: Logger; + user: { id?: UUID; name?: string }; conversationIndex: string; existingConversation: ConversationResponse; - conversation: ConversationUpdateProps; + conversationUpdateProps: ConversationUpdateProps; isPatch?: boolean; } @@ -57,11 +61,12 @@ export const updateConversation = async ({ logger, conversationIndex, existingConversation, - conversation, + conversationUpdateProps, isPatch, + user, }: UpdateConversationParams): Promise => { const updatedAt = new Date().toISOString(); - const params = transformToUpdateScheme(updatedAt, conversation); + const params = transformToUpdateScheme(updatedAt, conversationUpdateProps); try { const response = await esClient.updateByQuery({ @@ -69,7 +74,7 @@ export const updateConversation = async ({ index: conversationIndex, query: { ids: { - values: [existingConversation.id ?? ''], + values: [existingConversation.id], }, }, refresh: true, @@ -140,16 +145,19 @@ export const updateConversation = async ({ if (!response.updated && response.updated === 0) { throw Error('No conversation has been updated'); } + + const updatedConversation = await getConversation({ + esClient, + conversationIndex, + id: existingConversation.id, + logger, + user, + }); + return updatedConversation; } catch (err) { logger.warn(`Error updating conversation: ${err} by ID: ${existingConversation.id}`); throw err; } - const updatedConversation = await getConversation( - esClient, - conversationIndex, - existingConversation.id ?? '' - ); - return updatedConversation; }; export const transformToUpdateScheme = ( diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts index 0c6948811f178..d22fc44743384 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts @@ -57,7 +57,13 @@ export const appendConversationMessageRoute = (router: ElasticAssistantPluginRou existingConversation, request.body.messages ); - return response.ok({ body: conversation ?? {} }); + if (conversation == null) { + return assistantResponse.error({ + body: `conversation id: "${id}" was not updated with appended message`, + statusCode: 400, + }); + } + return response.ok({ body: conversation }); } catch (err) { const error = transformError(err); return assistantResponse.error({ diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts index ab92221134fa3..d0162282cc8b1 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts @@ -46,11 +46,10 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const currentUser = ctx.elasticAssistant.getCurrentUser(); - const additionalFilter = `title:${request.body.title}`; const result = await dataClient?.findConversations({ perPage: 100, page: 1, - filter: `user.id:${currentUser?.profile_uid}${additionalFilter}`, + filter: `user.id:${currentUser?.profile_uid} AND title:${request.body.title}`, fields: ['title'], }); if (result?.data != null && result.data.length > 0) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts index ac16ec91e7bf7..371a6aad6f49a 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts @@ -5,14 +5,12 @@ * 2.0. */ -import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, } from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; -import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { DeleteConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; import { buildRouteValidationWithZod } from '../route_validation'; @@ -35,7 +33,7 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => }, }, }, - async (context, request, response): Promise> => { + async (context, request, response) => { const assistantResponse = buildResponse(response); try { const { id } = request.params; @@ -52,7 +50,7 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => } await dataClient?.deleteConversation(id); - return response.ok({ body: {} }); + return response.ok(); } catch (err) { const error = transformError(err); return assistantResponse.error({ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts index c71442f6048fd..77a7dc17ba076 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -79,6 +79,9 @@ export const TraceData = z.object({ export type Replacement = z.infer; export const Replacement = z.object({}).catchall(z.unknown()); +export type Reader = z.infer; +export const Reader = z.object({}).catchall(z.unknown()); + /** * AI assistant conversation message. */ @@ -91,8 +94,8 @@ export const Message = z.object({ /** * Message content. */ - reader: z.string().optional(), - replacements: z.array(Replacement).optional(), + reader: Reader.optional(), + replacements: Replacement.optional(), /** * Message role. */ @@ -152,11 +155,11 @@ export const ErrorSchema = z export type ConversationResponse = z.infer; export const ConversationResponse = z.object({ - id: z.union([UUID, NonEmptyString]).optional(), + id: z.union([UUID, NonEmptyString]), /** * The conversation title. */ - title: z.string().optional(), + title: z.string(), timestamp: NonEmptyString.optional(), /** * The last time conversation was updated. @@ -165,9 +168,9 @@ export const ConversationResponse = z.object({ /** * The last time conversation was updated. */ - createdAt: z.string().optional(), - replacements: z.array(Replacement).optional(), - user: User.optional(), + createdAt: z.string(), + replacements: Replacement.optional(), + user: User, /** * The conversation messages. */ @@ -175,7 +178,7 @@ export const ConversationResponse = z.object({ /** * LLM API configuration. */ - apiConfig: ApiConfig.optional(), + apiConfig: ApiConfig, /** * Is default conversation. */ @@ -187,7 +190,7 @@ export const ConversationResponse = z.object({ /** * Kibana space */ - namespace: z.string().optional(), + namespace: z.string(), }); export type ConversationUpdateProps = z.infer; @@ -209,7 +212,7 @@ export const ConversationUpdateProps = z.object({ * excludeFromLastConversationStorage. */ excludeFromLastConversationStorage: z.boolean().optional(), - replacements: z.array(Replacement).optional(), + replacements: Replacement.optional(), }); export type ConversationCreateProps = z.infer; @@ -234,7 +237,7 @@ export const ConversationCreateProps = z.object({ * excludeFromLastConversationStorage. */ excludeFromLastConversationStorage: z.boolean().optional(), - replacements: z.array(Replacement).optional(), + replacements: Replacement.optional(), }); export type ConversationMessageCreateProps = z.infer; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml index c16054062a6e3..d28272a41228b 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -54,6 +54,10 @@ components: type: object additionalProperties: true + Reader: + type: object + additionalProperties: true + Message: type: object description: AI assistant conversation message. @@ -66,12 +70,10 @@ components: type: string description: Message content. reader: - type: string + $ref: '#/components/schemas/Reader' description: Message content. replacements: - type: array - items: - $ref: '#/components/schemas/Replacement' + $ref: '#/components/schemas/Replacement' role: type: string description: Message role. @@ -136,6 +138,13 @@ components: ConversationResponse: type: object + required: + - id + - title + - createdAt + - user + - namespace + - apiConfig properties: id: oneOf: @@ -153,9 +162,7 @@ components: description: The last time conversation was updated. type: string replacements: - type: array - items: - $ref: '#/components/schemas/Replacement' + $ref: '#/components/schemas/Replacement' user: $ref: '#/components/schemas/User' messages: @@ -200,9 +207,7 @@ components: description: excludeFromLastConversationStorage. type: boolean replacements: - type: array - items: - $ref: '#/components/schemas/Replacement' + $ref: '#/components/schemas/Replacement' ConversationCreateProps: type: object @@ -227,9 +232,7 @@ components: description: excludeFromLastConversationStorage. type: boolean replacements: - type: array - items: - $ref: '#/components/schemas/Replacement' + $ref: '#/components/schemas/Replacement' ConversationMessageCreateProps: type: object diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 380a456fb9875..5bc05de5295e5 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -4,20 +4,23 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useEffect } from 'react'; +import React, { useCallback, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import type { IToasts } from '@kbn/core-notifications-browser'; import type { Conversation } from '@kbn/elastic-assistant'; import { AssistantProvider as ElasticAssistantProvider, bulkChangeConversations, + mergeBaseWithPersistedConversations, + useFetchCurrentUserConversations, } from '@kbn/elastic-assistant'; +import type { FetchConversationsResponse } from '@kbn/elastic-assistant/impl/assistant/api'; import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; import { LOCAL_STORAGE_KEY, augmentMessageCodeBlocks } from './helpers'; -import { useBaseConversations, useConversationStore } from './use_conversation_store'; +import { useBaseConversations } from './use_conversation_store'; import { DEFAULT_ALLOW, DEFAULT_ALLOW_REPLACEMENT } from './content/anonymization'; import { PROMPT_CONTEXTS } from './content/prompt_contexts'; import { BASE_SECURITY_QUICK_PROMPTS } from './content/quick_prompts'; @@ -44,7 +47,16 @@ export const AssistantProvider: React.FC = ({ children }) => { const basePath = useBasePath(); const baseConversations = useBaseConversations(); - const userConversations = useConversationStore(); + const onFetchedConversations = useCallback( + (conversationsData: FetchConversationsResponse): Record => + mergeBaseWithPersistedConversations({}, conversationsData), + [] + ); + const { + data: conversationsData, + isLoading, + isError, + } = useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); const assistantAvailability = useAssistantAvailability(); const assistantTelemetry = useAssistantTelemetry(); @@ -62,7 +74,10 @@ export const AssistantProvider: React.FC = ({ children }) => { useEffect(() => { const migrateConversationsFromLocalStorage = async () => { if ( - Object.keys(userConversations).length > 0 && + !isLoading && + !isError && + conversationsData && + Object.keys(conversationsData).length === 0 && conversations && Object.keys(conversations).length > 0 ) { @@ -82,7 +97,7 @@ export const AssistantProvider: React.FC = ({ children }) => { } }; migrateConversationsFromLocalStorage(); - }, [conversations, http, storage, userConversations]); + }, [conversations, conversationsData, http, isError, isLoading, storage]); return ( => { - const [conversations, setConversations] = useState>({}); - const { - services: { http }, - } = useKibana(); - const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); - const baseConversations = useMemo( - () => - isDataQualityDashboardPageExists - ? BASE_SECURITY_CONVERSATIONS - : unset(DATA_QUALITY_DASHBOARD_CONVERSATION_ID, BASE_SECURITY_CONVERSATIONS), - [isDataQualityDashboardPageExists] - ); - - const onFetchedConversations = useCallback( - (conversationsData: FetchConversationsResponse): Record => { - const userConversations = (conversationsData?.data ?? []).reduce< - Record - >((transformed, conversation) => { - transformed[conversation.id] = conversation; - return transformed; - }, {}); - return merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => - (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ); - }, - [baseConversations] - ); - const { - data: conversationsData, - isLoading, - isError, - } = useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); - - useEffect(() => { - if (!isLoading && !isError) { - setConversations(conversationsData ?? {}); - } - }, [conversationsData, isLoading, isError]); - - const result = useMemo( - () => merge(baseConversations, conversations), - [baseConversations, conversations] - ); - - return result; -}; export const useBaseConversations = (): Record => { const isDataQualityDashboardPageExists = useLinkAuthorized(SecurityPageName.dataQuality); From 8e6de8b9d6a30da1fb47343f43e30cc5db6293e7 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 30 Jan 2024 22:29:46 -0800 Subject: [PATCH 056/141] resolved conflicts --- .../impl/assistant/api/index.test.tsx | 1 + .../impl/assistant/api/index.tsx | 9 +- .../server/__mocks__/request.ts | 17 +-- .../server/lib/model_evaluator/evaluation.ts | 1 - .../elastic_assistant/server/plugin.ts | 40 ------- .../routes/evaluate/post_evaluate.test.ts | 13 --- .../server/routes/evaluate/post_evaluate.ts | 41 +------ .../server/routes/evaluate/utils.ts | 4 - .../evaluate/post_evaluate_route.gen.ts | 54 --------- .../evaluate/post_evaluate_route.schema.yaml | 109 ------------------ .../plugins/elastic_assistant/tsconfig.json | 3 - 11 files changed, 6 insertions(+), 286 deletions(-) delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx index d6929f55dc30f..79a7cb91c37de 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx @@ -13,6 +13,7 @@ import { fetchConnectorExecuteAction, FetchConnectorExecuteAction, getKnowledgeBaseStatus, + postEvaluation, postKnowledgeBase, } from '.'; import type { Conversation, Message } from '../../assistant_context/types'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index 1c27553702d44..88b1cbe2827cc 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -15,14 +15,10 @@ import { getFormattedMessageContent, getOptionalRequestParams, hasParsableResponse, -<<<<<<< HEAD:x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx } from '../helpers'; -import { PerformEvaluationParams } from '../settings/evaluation_settings/use_perform_evaluation'; +import { PerformEvaluationParams } from './evaluate/use_perform_evaluation'; export * from './conversations'; -======= -} from './helpers'; ->>>>>>> upstream/main:x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx export interface FetchConnectorExecuteAction { isEnabledRAGAlerts: boolean; @@ -348,7 +344,6 @@ export const deleteKnowledgeBase = async ({ return error as IHttpFetchError; } }; -<<<<<<< HEAD:x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx export interface PostEvaluationParams { http: HttpSetup; @@ -408,5 +403,3 @@ export const postEvaluation = async ({ return error as IHttpFetchError; } }; -======= ->>>>>>> upstream/main:x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 6338bb76e483f..dab6b37c51476 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -6,11 +6,6 @@ */ import { httpServerMock } from '@kbn/core/server/mocks'; import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants'; -import { -<<<<<<< HEAD - EvaluateRequestBodyInput, - EvaluateRequestQueryInput, -} from '../schemas/evaluate/post_evaluate_route.gen'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, @@ -18,6 +13,8 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + PostEvaluateRequestBodyInput, + PostEvaluateRequestQueryInput, } from '@kbn/elastic-assistant-common'; import { getAppendConversationMessagesSchemaMock, @@ -28,11 +25,6 @@ import { ConversationCreateProps, ConversationUpdateProps, } from '../schemas/conversations/common_attributes.gen'; -======= - PostEvaluateRequestBodyInput, - PostEvaluateRequestQueryInput, -} from '@kbn/elastic-assistant-common'; ->>>>>>> upstream/main export const requestMock = { create: httpServerMock.createKibanaRequest, @@ -69,13 +61,8 @@ export const getPostEvaluateRequest = ({ body, query, }: { -<<<<<<< HEAD - body: EvaluateRequestBodyInput; - query: EvaluateRequestQueryInput; -======= body: PostEvaluateRequestBodyInput; query: PostEvaluateRequestQueryInput; ->>>>>>> upstream/main }) => requestMock.create({ body, diff --git a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts index 3754868d8d685..db1566572584c 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts @@ -16,7 +16,6 @@ import { Dataset } from '@kbn/elastic-assistant-common'; import { AgentExecutorEvaluatorWithMetadata } from '../langchain/executors/types'; import { callAgentWithRetry, getMessageFromLangChainResponse } from './utils'; import { isLangSmithEnabled, writeLangSmithFeedback } from '../../routes/evaluate/utils'; -import { Dataset } from '../../schemas/evaluate/post_evaluate_route.gen'; import { ResponseBody } from '../langchain/types'; export interface PerformEvaluationParams { diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index c2b4cc0142ca8..0d3acd5f8f643 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -19,37 +19,12 @@ import { ElasticAssistantPluginStartDependencies, ElasticAssistantRequestHandlerContext, } from './types'; -<<<<<<< HEAD import { AIAssistantService } from './ai_assistant_service'; import { assistantPromptsType, assistantAnonimizationFieldsType } from './saved_object'; import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; import { registerRoutes } from './routes/register_routes'; import { appContextService } from './services/app_context'; -======= -import { - deleteKnowledgeBaseRoute, - getKnowledgeBaseStatusRoute, - postActionsConnectorExecuteRoute, - postEvaluateRoute, - postKnowledgeBaseRoute, -} from './routes'; -import { - appContextService, - GetRegisteredFeatures, - GetRegisteredTools, -} from './services/app_context'; -import { getCapabilitiesRoute } from './routes/capabilities/get_capabilities_route'; -import { getEvaluateRoute } from './routes/evaluate/get_evaluate'; - -interface CreateRouteHandlerContextParams { - core: CoreSetup; - logger: Logger; - getRegisteredFeatures: GetRegisteredFeatures; - getRegisteredTools: GetRegisteredTools; - telemetry: AnalyticsServiceSetup; -} ->>>>>>> upstream/main export class ElasticAssistantPlugin implements @@ -107,21 +82,6 @@ export class ElasticAssistantPlugin // this.assistantService registerKBTask registerRoutes(router, this.logger, plugins); - -<<<<<<< HEAD -======= - // Knowledge Base - deleteKnowledgeBaseRoute(router); - getKnowledgeBaseStatusRoute(router, getElserId); - postKnowledgeBaseRoute(router, getElserId); - // Actions Connector Execute (LLM Wrapper) - postActionsConnectorExecuteRoute(router, getElserId); - // Evaluate - postEvaluateRoute(router, getElserId); - getEvaluateRoute(router); - // Capabilities - getCapabilitiesRoute(router); ->>>>>>> upstream/main return { actions: plugins.actions, getRegisteredFeatures: (pluginName: string) => { diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts index f6a44d7aa420b..64ec69fa5e943 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts @@ -9,30 +9,17 @@ import { postEvaluateRoute } from './post_evaluate'; import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getPostEvaluateRequest } from '../../__mocks__/request'; -<<<<<<< HEAD -import { - EvaluateRequestQueryInput, - EvaluateRequestBodyInput, -} from '../../schemas/evaluate/post_evaluate_route.gen'; - -const defaultBody: EvaluateRequestBodyInput = { -======= import type { PostEvaluateRequestBodyInput, PostEvaluateRequestQueryInput, } from '@kbn/elastic-assistant-common'; const defaultBody: PostEvaluateRequestBodyInput = { ->>>>>>> upstream/main dataset: undefined, evalPrompt: undefined, }; -<<<<<<< HEAD -const defaultQueryParams: EvaluateRequestQueryInput = { -======= const defaultQueryParams: PostEvaluateRequestQueryInput = { ->>>>>>> upstream/main agents: 'agents', datasetName: undefined, evaluationType: undefined, diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index 3612db646a49b..33ada7d338b05 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -9,9 +9,6 @@ import { type IKibanaResponse, IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { v4 as uuidv4 } from 'uuid'; -<<<<<<< HEAD -import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; -======= import { API_VERSIONS, INTERNAL_API_ACCESS, @@ -19,7 +16,6 @@ import { PostEvaluateRequestQuery, PostEvaluateResponse, } from '@kbn/elastic-assistant-common'; ->>>>>>> upstream/main import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; @@ -33,26 +29,14 @@ import { } from '../../lib/model_evaluator/output_index/utils'; import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -<<<<<<< HEAD -import { - EvaluateRequestBody, - EvaluateRequestQuery, -} from '../../schemas/evaluate/post_evaluate_route.gen'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; /** * To support additional Agent Executors from the UI, add them to this map * and reference your specific AgentExecutor function */ -const AGENT_EXECUTOR_MAP: Record = { - DefaultAgentExecutor: callAgentExecutor, - OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor, -}; -======= import { buildRouteValidationWithZod } from '../../schemas/common'; import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; ->>>>>>> upstream/main +import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; const DEFAULT_SIZE = 20; @@ -62,31 +46,15 @@ export const postEvaluateRoute = ( ) => { router.versioned .post({ -<<<<<<< HEAD - access: 'internal', - path: EVALUATE, - -======= access: INTERNAL_API_ACCESS, path: EVALUATE, ->>>>>>> upstream/main + options: { tags: ['access:elasticAssistant'], }, }) .addVersion( { -<<<<<<< HEAD - version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, - validate: { - request: { - body: buildRouteValidationWithZod(EvaluateRequestBody), - query: buildRouteValidationWithZod(EvaluateRequestQuery), - }, - }, - }, - async (context, request, response) => { -======= version: API_VERSIONS.internal.v1, validate: { request: { @@ -101,7 +69,6 @@ export const postEvaluateRoute = ( }, }, async (context, request, response): Promise> => { ->>>>>>> upstream/main const assistantContext = await context.elasticAssistant; const logger = assistantContext.logger; const telemetry = assistantContext.telemetry; @@ -175,11 +142,7 @@ export const postEvaluateRoute = ( // Skeleton request from route to pass to the agents // params will be passed to the actions executor -<<<<<<< HEAD const skeletonRequest: KibanaRequest = { -======= - const skeletonRequest: KibanaRequest = { ->>>>>>> upstream/main ...request, body: { alertsIndexPattern: '', diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts index 984f4203683ae..11f8cb9c2f692 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts @@ -12,11 +12,7 @@ import type { Logger } from '@kbn/core/server'; import type { Run } from 'langsmith/schemas'; import { ToolingLog } from '@kbn/tooling-log'; import { LangChainTracer } from 'langchain/callbacks'; -<<<<<<< HEAD -import { Dataset } from '../../schemas/evaluate/post_evaluate_route.gen'; -======= import { Dataset } from '@kbn/elastic-assistant-common'; ->>>>>>> upstream/main /** * Returns the LangChain `llmType` for the given connectorId/connectors diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts deleted file mode 100644 index 306922f7242f3..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.gen.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from 'zod'; - -/* - * NOTICE: Do not edit this file manually. - * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. - * - * info: - * title: Evaluate API endpoint - * version: 2023-10-31 - */ - -export type DatasetItem = z.infer; -export const DatasetItem = z.object({ - id: z.string(), - input: z.string(), - reference: z.string(), - tags: z.array(z.string()).optional(), - prediction: z.string().optional(), -}); - -export type Dataset = z.infer; -export const Dataset = z.array(DatasetItem); - -export type EvaluateRequestQuery = z.infer; -export const EvaluateRequestQuery = z.object({ - agents: z.string(), - models: z.string(), - outputIndex: z.string(), - datasetName: z.string().optional(), - evaluationType: z.string().optional(), - evalModel: z.string().optional(), - projectName: z.string().optional(), - runName: z.string().optional(), -}); -export type EvaluateRequestQueryInput = z.input; - -export type EvaluateRequestBody = z.infer; -export const EvaluateRequestBody = z.object({ - dataset: Dataset.optional(), - evalPrompt: z.string().optional(), -}); -export type EvaluateRequestBodyInput = z.input; - -export type EvaluateResponse = z.infer; -export const EvaluateResponse = z.object({ - evaluationId: z.number().optional(), -}); diff --git a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml deleted file mode 100644 index 209c6d748cb27..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate_route.schema.yaml +++ /dev/null @@ -1,109 +0,0 @@ -openapi: 3.0.0 -info: - title: Evaluate API endpoint - version: '2023-10-31' -paths: - /internal/elastic_assistant/evaluate: - get: - operationId: Evaluate - x-codegen-enabled: true - description: Get Elastic Assistant capabilities for the requesting plugin - summary: Get Elastic Assistant capabilities - tags: - - Evaluate API - parameters: - - in: query - name: agents - required: true - schema: - type: string - - in: query - name: models - required: true - schema: - type: string - - in: query - name: outputIndex - required: true - schema: - type: string - - in: query - name: datasetName - schema: - type: string - - in: query - name: evaluationType - schema: - type: string - - in: query - name: evalModel - schema: - type: string - - in: query - name: projectName - schema: - type: string - - in: query - name: runName - schema: - type: string - requestBody: - content: - application/json: - schema: - type: object - properties: - dataset: - $ref: '#/components/schemas/Dataset' - evalPrompt: - type: string - - responses: - 200: - description: Successful response - content: - application/json: - schema: - type: object - properties: - evaluationId: - type: number - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string - -components: - schemas: - DatasetItem: - type: object - required: - - id - - reference - - input - properties: - id: - type: string - input: - type: string - reference: - type: string - tags: - type: array - items: - type: string - prediction: - type: string - Dataset: - type: array - items: - $ref: '#/components/schemas/DatasetItem' diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 21bf75fc670b3..48b3ac39e239a 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -34,7 +34,6 @@ "@kbn/core-analytics-server", "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", -<<<<<<< HEAD "@kbn/data-stream-adapter", "@kbn/alerts-as-data-utils", "@kbn/core-saved-objects-utils-server", @@ -47,9 +46,7 @@ "@kbn/zod-helpers", "@kbn/core-saved-objects-server", "@kbn/spaces-plugin", -======= "@kbn/zod-helpers", ->>>>>>> upstream/main ], "exclude": [ "target/**/*", From 76da77804f43bfdf1b0732bd7c0f9c63a7b9b022 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 31 Jan 2024 06:38:10 +0000 Subject: [PATCH 057/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/elastic_assistant/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 48b3ac39e239a..5ba0fea4969cb 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -47,6 +47,7 @@ "@kbn/core-saved-objects-server", "@kbn/spaces-plugin", "@kbn/zod-helpers", + "@kbn/securitysolution-io-ts-utils", ], "exclude": [ "target/**/*", From 1e074d455f555dc71ec350404f6da720d876718f Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 31 Jan 2024 13:38:33 -0800 Subject: [PATCH 058/141] Fixed failing tests --- .../use_bulk_actions_conversations.ts | 2 +- .../conversation_selector/index.test.tsx | 25 ++++-- .../__mocks__/conversations_schema.mock.ts | 2 + .../create_conversation.test.ts | 78 ++++++++++++++++++ .../create_conversation.ts | 79 +++++++------------ .../delete_conversation.test.ts | 12 ++- .../delete_conversation.ts | 9 ++- .../get_conversation.test.ts | 24 ++++-- .../server/conversations_data_client/index.ts | 10 +-- .../server/conversations_data_client/types.ts | 17 ++-- .../update_conversation.test.ts | 22 +++++- .../update_conversation.ts | 6 +- .../routes/conversations/create_route.ts | 7 ++ .../routes/conversations/delete_route.ts | 2 +- .../conversations/common_attributes.gen.ts | 20 ++++- .../common_attributes.schema.yaml | 26 +++--- 16 files changed, 246 insertions(+), 95 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 089872a4dc220..e19e24988b094 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -119,7 +119,7 @@ export const bulkChangeConversations = ( body: JSON.stringify({ update: conversationsToUpdate, create: conversationsToCreate, - delete: conversationsActions.delete?.ids, + delete: conversationsActions.delete, }), }); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx index b13164e1570b5..27caa9d0dfb2b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx @@ -75,7 +75,10 @@ describe('Conversation selector', () => { ); fireEvent.click(getByTestId('comboBoxSearchInput')); fireEvent.click(getByTestId(`convo-option-${alertConvo.id}`)); - expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id, alertConvo.title); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: alertConvo.id, + cTitle: alertConvo.title, + }); }); it('On clear input, clears selected options', () => { const { getByPlaceholderText, queryByPlaceholderText, getByTestId, queryByTestId } = render( @@ -103,7 +106,10 @@ describe('Conversation selector', () => { code: 'Enter', charCode: 13, }); - expect(onConversationSelected).toHaveBeenCalledWith(customOption, customOption); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: customOption, + cTitle: customOption, + }); }); it('Only custom options can be deleted', () => { @@ -157,7 +163,10 @@ describe('Conversation selector', () => { fireEvent.click( within(getByTestId(`convo-option-${customConvo.id}`)).getByTestId('delete-option') ); - expect(onConversationSelected).toHaveBeenCalledWith(welcomeConvo.id, welcomeConvo.title); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: welcomeConvo.id, + cTitle: welcomeConvo.title, + }); }); it('Left arrow selects first conversation', () => { @@ -175,7 +184,10 @@ describe('Conversation selector', () => { code: 'ArrowLeft', charCode: 27, }); - expect(onConversationSelected).toHaveBeenCalledWith(alertConvo.id, alertConvo.title); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: alertConvo.id, + cTitle: alertConvo.title, + }); }); it('Right arrow selects last conversation', () => { @@ -191,7 +203,10 @@ describe('Conversation selector', () => { code: 'ArrowRight', charCode: 26, }); - expect(onConversationSelected).toHaveBeenCalledWith(customConvo.id, customConvo.title); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: customConvo.id, + cTitle: customConvo.title, + }); }); it('Right arrow does nothing when ctrlKey is false', () => { diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 8556fb34f9e5d..47d9cb4908207 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -80,6 +80,8 @@ export const getConversationMock = ( params: ConversationCreateProps | ConversationUpdateProps ): ConversationResponse => ({ id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + apiConfig: {}, + title: 'test', ...params, createdAt: '2019-12-13T16:40:33.400Z', updatedAt: '2019-12-13T16:40:33.400Z', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index b6a037322f2ad..e0604d8d10775 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -11,6 +11,14 @@ import { ConversationCreateProps, ConversationResponse, } from '../schemas/conversations/common_attributes.gen'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { estypes } from '@elastic/elasticsearch'; +import { SearchEsConversationSchema } from './types'; +import { getConversation } from './get_conversation'; + +jest.mock('./get_conversation', () => ({ + getConversation: jest.fn(), +})); export const getCreateConversationMock = (): ConversationCreateProps => ({ title: 'test', @@ -52,9 +60,58 @@ export const getConversationResponseMock = (): ConversationResponse => ({ }, }); +export const getSearchConversationMock = + (): estypes.SearchResponse => ({ + _scroll_id: '123', + _shards: { + failed: 0, + skipped: 0, + successful: 0, + total: 0, + }, + hits: { + hits: [ + { + _id: '1', + _index: '', + _score: 0, + _source: { + '@timestamp': '2020-04-20T15:25:31.830Z', + created_at: '2020-04-20T15:25:31.830Z', + title: 'title-1', + updated_at: '2020-04-20T15:25:31.830Z', + messages: [], + id: '1', + namespace: 'default', + is_default: true, + exclude_from_last_conversation_storage: false, + api_config: { + connector_id: 'c1', + connector_type_title: 'title-c-1', + default_system_prompt_id: 'prompt-1', + model: 'test', + provider: 'Azure OpenAI', + }, + user: { + id: '1111', + name: 'elastic', + }, + replacements: undefined, + }, + }, + ], + max_score: 0, + total: 1, + }, + timed_out: false, + took: 10, + }); + describe('createConversation', () => { + let logger: ReturnType; beforeEach(() => { jest.clearAllMocks(); + logger = loggingSystemMock.createLogger(); }); afterEach(() => { @@ -73,6 +130,10 @@ describe('createConversation', () => { test('it returns a conversation as expected with the id changed out for the elastic id', async () => { const conversation = getCreateConversationMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce({ + ...getConversationResponseMock(), + id: 'elastic-id-123', + }); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( @@ -85,6 +146,7 @@ describe('createConversation', () => { spaceId: 'test', user: { name: 'test' }, conversation, + logger, }); const expected: ConversationResponse = { @@ -100,6 +162,12 @@ describe('createConversation', () => { ...getCreateConversationMock(), title: 'test new title', }; + (getConversation as unknown as jest.Mock).mockResolvedValueOnce({ + ...getConversationResponseMock(), + id: 'elastic-id-123', + title: 'test new title', + }); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface @@ -111,6 +179,7 @@ describe('createConversation', () => { spaceId: 'test', user: { name: 'test' }, conversation, + logger, }); const expected: ConversationResponse = { @@ -123,6 +192,8 @@ describe('createConversation', () => { test('It calls "esClient" with body, id, and conversationIndex', async () => { const conversation = getCreateConversationMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce(getConversationResponseMock()); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; await createConversation({ esClient, @@ -130,6 +201,7 @@ describe('createConversation', () => { spaceId: 'test', user: { name: 'test' }, conversation, + logger, }); expect(esClient.create).toBeCalled(); @@ -137,6 +209,11 @@ describe('createConversation', () => { test('It returns an auto-generated id if id is sent in undefined', async () => { const conversation = getCreateConversationMock(); + (getConversation as unknown as jest.Mock).mockResolvedValueOnce({ + ...getConversationResponseMock(), + id: 'elastic-id-123', + }); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.create.mockResponse( // @ts-expect-error not full response interface @@ -148,6 +225,7 @@ describe('createConversation', () => { spaceId: 'test', user: { name: 'test' }, conversation, + logger, }); const expected: ConversationResponse = { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 56e8d0390a7f8..1920874c90d64 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -6,15 +6,18 @@ */ import { v4 as uuidv4 } from 'uuid'; -import { ElasticsearchClient } from '@kbn/core/server'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { ConversationCreateProps, ConversationResponse, + MessageRole, + Provider, Reader, Replacement, UUID, } from '../schemas/conversations/common_attributes.gen'; +import { getConversation } from './get_conversation'; export interface CreateMessageSchema { '@timestamp'?: string; @@ -26,7 +29,7 @@ export interface CreateMessageSchema { content: string; reader?: Reader; replacements?: Replacement; - role: 'user' | 'assistant' | 'system'; + role: MessageRole; is_error?: boolean; presentation?: { delay?: number; @@ -41,7 +44,7 @@ export interface CreateMessageSchema { connector_id?: string; connector_type_title?: string; default_system_prompt_id?: string; - provider?: 'OpenAI' | 'Azure OpenAI'; + provider?: Provider; model?: string; }; is_default?: boolean; @@ -57,6 +60,7 @@ export interface CreateMessageSchema { export interface CreateConversationParams { esClient: ElasticsearchClient; + logger: Logger; conversationIndex: string; spaceId: string; user: { id?: UUID; name?: string }; @@ -69,21 +73,30 @@ export const createConversation = async ({ spaceId, user, conversation, -}: CreateConversationParams): Promise => { + logger, +}: CreateConversationParams): Promise => { const createdAt = new Date().toISOString(); const body = transformToCreateScheme(createdAt, spaceId, user, conversation); + try { + const response = await esClient.create({ + body, + id: uuidv4(), + index: conversationIndex, + refresh: 'wait_for', + }); - const response = await esClient.create({ - body, - id: uuidv4(), - index: conversationIndex, - refresh: 'wait_for', - }); - - return { - ...transform(body), - id: response._id, - }; + const createdConversation = await getConversation({ + esClient, + conversationIndex, + id: response._id, + logger, + user, + }); + return createdConversation; + } catch (err) { + logger.warn(`Error creating conversation: ${err} with title: ${conversation.title}`); + throw err; + } }; export const transformToCreateScheme = ( @@ -98,7 +111,7 @@ export const transformToCreateScheme = ( messages, replacements, }: ConversationCreateProps -) => { +): CreateMessageSchema => { return { '@timestamp': createdAt, created_at: createdAt, @@ -131,37 +144,3 @@ export const transformToCreateScheme = ( namespace: spaceId, }; }; - -function transform(conversationSchema: CreateMessageSchema): Omit { - return { - timestamp: conversationSchema['@timestamp'], - createdAt: conversationSchema.created_at, - user: conversationSchema.user, - title: conversationSchema.title, - apiConfig: { - connectorId: conversationSchema.api_config?.connector_id, - connectorTypeTitle: conversationSchema.api_config?.connector_type_title, - defaultSystemPromptId: conversationSchema.api_config?.default_system_prompt_id, - model: conversationSchema.api_config?.model, - provider: conversationSchema.api_config?.provider as 'OpenAI' | 'Azure OpenAI' | undefined, - }, - excludeFromLastConversationStorage: conversationSchema.exclude_from_last_conversation_storage, - isDefault: conversationSchema.is_default, - messages: conversationSchema.messages?.map((message) => ({ - timestamp: message['@timestamp'], - content: message.content, - isError: message.is_error, - presentation: message.presentation, - reader: message.reader, - replacements: message.replacements as Replacement, - role: message.role, - traceData: { - traceId: message.trace_data?.trace_id, - transactionId: message.trace_data?.transaction_id, - }, - })), - updatedAt: conversationSchema.updated_at, - replacements: conversationSchema.replacements as Replacement, - namespace: conversationSchema.namespace, - }; -} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index b1f117e4caecf..1bf20217aaebd 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -9,6 +9,7 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m import { DeleteConversationParams, deleteConversation } from './delete_conversation'; import { getConversation } from './get_conversation'; import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; jest.mock('./get_conversation', () => ({ getConversation: jest.fn(), @@ -33,12 +34,21 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: Date.now().toLocaleString(), timestamp: Date.now().toLocaleString(), + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }); export const getDeleteConversationOptionsMock = (): DeleteConversationParams => ({ esClient: elasticsearchClientMock.createScopedClusterClient().asCurrentUser, id: 'test', - conversationIndex: '', + conversationIndex: '.kibana-elastic-ai-assistant-conversations', + logger: loggingSystemMock.createLogger(), + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }); describe('deleteConversation', () => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index f824b35172fbe..06079def47947 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -5,20 +5,25 @@ * 2.0. */ -import { ElasticsearchClient } from '@kbn/core/server'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { getConversation } from './get_conversation'; +import { UUID } from '../schemas/conversations/common_attributes.gen'; export interface DeleteConversationParams { esClient: ElasticsearchClient; conversationIndex: string; id: string; + logger: Logger; + user: { id?: UUID; name?: string }; } export const deleteConversation = async ({ esClient, conversationIndex, id, + logger, + user, }: DeleteConversationParams): Promise => { - const conversation = await getConversation(esClient, conversationIndex, id); + const conversation = await getConversation({ esClient, conversationIndex, id, logger, user }); if (conversation !== null) { const response = await esClient.deleteByQuery({ body: { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index e21d2e994a342..5820307ffa0d1 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -5,11 +5,13 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { getConversation } from './get_conversation'; import { estypes } from '@elastic/elasticsearch'; import { SearchEsConversationSchema } from './types'; import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; export const getConversationResponseMock = (): ConversationResponse => ({ createdAt: '2020-04-20T15:25:31.830Z', @@ -83,8 +85,10 @@ export const getSearchConversationMock = }); describe('getConversation', () => { + let loggerMock: Logger; beforeEach(() => { jest.clearAllMocks(); + loggerMock = loggingSystemMock.createLogger(); }); afterEach(() => { @@ -95,11 +99,13 @@ describe('getConversation', () => { const data = getSearchConversationMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation( + const conversation = await getConversation({ esClient, - '.kibana-elastic-ai-assistant-conversations', - '1' - ); + conversationIndex: '.kibana-elastic-ai-assistant-conversations', + id: '1', + logger: loggerMock, + user: { name: 'test' }, + }); const expected = getConversationResponseMock(); expect(conversation).toEqual(expected); }); @@ -109,11 +115,13 @@ describe('getConversation', () => { data.hits.hits = []; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.search.mockResponse(data); - const conversation = await getConversation( + const conversation = await getConversation({ esClient, - '.kibana-elastic-ai-assistant-conversations', - '1' - ); + conversationIndex: '.kibana-elastic-ai-assistant-conversations', + id: '1', + logger: loggerMock, + user: { name: 'test' }, + }); expect(conversation).toEqual(null); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index d032f24935e9e..dd2bbcf869dab 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -28,11 +28,6 @@ import { getConversation } from './get_conversation'; import { deleteConversation } from './delete_conversation'; import { appendConversationMessages } from './append_conversation_messages'; -export enum OpenAiProviderType { - OpenAi = 'OpenAI', - AzureAi = 'Azure OpenAI', -} - export interface AIAssistantConversationsDataClientParams { elasticsearchClientPromise: Promise; kibanaVersion: string; @@ -188,11 +183,12 @@ export class AIAssistantConversationsDataClient { */ public createConversation = async ( conversation: ConversationCreateProps - ): Promise => { + ): Promise => { const { currentUser } = this; const esClient = await this.options.elasticsearchClientPromise; return createConversation({ esClient, + logger: this.options.logger, conversationIndex: this.indexTemplateAndPattern.alias, spaceId: this.spaceId, user: { id: currentUser?.profile_uid, name: currentUser?.username }, @@ -241,6 +237,8 @@ export class AIAssistantConversationsDataClient { esClient, conversationIndex: this.indexTemplateAndPattern.alias, id, + logger: this.options.logger, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, }); }; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts index 999dd03d3e5fa..8ecc8e0568537 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts @@ -5,6 +5,13 @@ * 2.0. */ +import { + MessageRole, + Provider, + Reader, + Replacement, +} from '../schemas/conversations/common_attributes.gen'; + export interface SearchEsConversationSchema { id: string; '@timestamp': string; @@ -13,9 +20,9 @@ export interface SearchEsConversationSchema { messages?: Array<{ '@timestamp': string; content: string; - reader?: string | undefined; - replacements?: unknown; - role: 'user' | 'assistant' | 'system'; + reader?: Reader; + replacements?: Replacement; + role: MessageRole; is_error?: boolean; presentation?: { delay?: number; @@ -30,12 +37,12 @@ export interface SearchEsConversationSchema { connector_id?: string; connector_type_title?: string; default_system_prompt_id?: string; - provider?: 'OpenAI' | 'Azure OpenAI'; + provider?: Provider; model?: string; }; is_default?: boolean; exclude_from_last_conversation_storage?: boolean; - replacements?: unknown; + replacements?: Replacement; user?: { id?: string; name?: string; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 60d645098e30a..221dda2ccc435 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -49,6 +49,10 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: '2020-04-20T15:25:31.830Z', timestamp: '2020-04-20T15:25:31.830Z', + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }); jest.mock('./get_conversation', () => ({ @@ -77,7 +81,11 @@ describe('updateConversation', () => { logger: loggerMock.create(), conversationIndex: 'index-1', existingConversation, - conversation, + conversationUpdateProps: conversation, + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }); const expected: ConversationResponse = { ...getConversationResponseMock(), @@ -98,7 +106,11 @@ describe('updateConversation', () => { logger: loggerMock.create(), conversationIndex: 'index-1', existingConversation, - conversation, + conversationUpdateProps: conversation, + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }); expect(updatedList).toEqual(null); }); @@ -115,7 +127,11 @@ describe('updateConversation', () => { logger: loggerMock.create(), conversationIndex: 'index-1', existingConversation, - conversation, + conversationUpdateProps: conversation, + user: { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, }) ).rejects.toThrow('No conversation has been updated'); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index a4e1f7744a65c..2b734e9cf3e5a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -12,6 +12,8 @@ import { Reader, ConversationUpdateProps, UUID, + Provider, + MessageRole, } from '../schemas/conversations/common_attributes.gen'; import { getConversation } from './get_conversation'; @@ -23,7 +25,7 @@ export interface UpdateConversationSchema { content: string; reader?: Reader; replacements?: Replacement; - role: 'user' | 'assistant' | 'system'; + role: MessageRole; is_error?: boolean; presentation?: { delay?: number; @@ -38,7 +40,7 @@ export interface UpdateConversationSchema { connector_id?: string; connector_type_title?: string; default_system_prompt_id?: string; - provider?: 'OpenAI' | 'Azure OpenAI'; + provider?: Provider; model?: string; }; exclude_from_last_conversation_storage?: boolean; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts index d0162282cc8b1..87bf34e80a425 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts @@ -59,6 +59,13 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v }); } const createdConversation = await dataClient?.createConversation(request.body); + + if (createdConversation == null) { + return assistantResponse.error({ + body: `conversation with title: "${request.body.title}" was not created`, + statusCode: 400, + }); + } return response.ok({ body: ConversationResponse.parse(createdConversation), }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts index 371a6aad6f49a..7e85d24bc8107 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts @@ -50,7 +50,7 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => } await dataClient?.deleteConversation(id); - return response.ok(); + return response.ok({ body: {} }); } catch (err) { const error = transformError(err); return assistantResponse.error({ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts index 77a7dc17ba076..87b656c4e0096 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts @@ -82,6 +82,22 @@ export const Replacement = z.object({}).catchall(z.unknown()); export type Reader = z.infer; export const Reader = z.object({}).catchall(z.unknown()); +/** + * Provider + */ +export type Provider = z.infer; +export const Provider = z.enum(['OpenAI', 'Azure OpenAI']); +export type ProviderEnum = typeof Provider.enum; +export const ProviderEnum = Provider.enum; + +/** + * Message role. + */ +export type MessageRole = z.infer; +export const MessageRole = z.enum(['system', 'user', 'assistant']); +export type MessageRoleEnum = typeof MessageRole.enum; +export const MessageRoleEnum = MessageRole.enum; + /** * AI assistant conversation message. */ @@ -99,7 +115,7 @@ export const Message = z.object({ /** * Message role. */ - role: z.enum(['system', 'user', 'assistant']), + role: MessageRole, /** * The timestamp message was sent or received. */ @@ -135,7 +151,7 @@ export const ApiConfig = z.object({ /** * Provider */ - provider: z.enum(['OpenAI', 'Azure OpenAI']).optional(), + provider: Provider.optional(), /** * model */ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml index d28272a41228b..e2208b8452c5b 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml @@ -58,6 +58,21 @@ components: type: object additionalProperties: true + Provider: + type: string + description: Provider + enum: + - OpenAI + - Azure OpenAI + + MessageRole: + type: string + description: Message role. + enum: + - system + - user + - assistant + Message: type: object description: AI assistant conversation message. @@ -75,12 +90,8 @@ components: replacements: $ref: '#/components/schemas/Replacement' role: - type: string + $ref: '#/components/schemas/MessageRole' description: Message role. - enum: - - system - - user - - assistant timestamp: $ref: '#/components/schemas/NonEmptyString' description: The timestamp message was sent or received. @@ -107,11 +118,8 @@ components: type: string description: defaultSystemPromptId provider: - type: string + $ref: '#/components/schemas/Provider' description: Provider - enum: - - OpenAI - - Azure OpenAI model: type: string description: model From 980a44e0a9c690b4265b731247ab6c3edae5ef62 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 31 Jan 2024 16:35:40 -0800 Subject: [PATCH 059/141] - --- .../assistant/get_comments/index.test.tsx | 1 + .../use_conversation_store/index.test.tsx | 48 ++----------------- .../open_and_acknowledged_alerts_tool.ts | 1 + 3 files changed, 6 insertions(+), 44 deletions(-) diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx index a942a1b8f0f4a..63bd155704553 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx @@ -12,6 +12,7 @@ const user: ConversationRole = 'user'; const currentConversation = { apiConfig: {}, id: '1', + title: '1', messages: [ { role: user, diff --git a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx index e71cedf8f955c..7fee44d0178b1 100644 --- a/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/use_conversation_store/index.test.tsx @@ -5,14 +5,13 @@ * 2.0. */ import { renderHook } from '@testing-library/react-hooks'; -import { useConversationStore } from '.'; +import { useBaseConversations } from '.'; import { useLinkAuthorized } from '../../common/links'; import { useKibana as mockUseKibana } from '../../common/lib/kibana/__mocks__'; import { DATA_QUALITY_DASHBOARD_CONVERSATION_ID } from '@kbn/ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/translations'; import { useKibana } from '../../common/lib/kibana'; import { BASE_SECURITY_CONVERSATIONS } from '../content/conversations'; import { unset } from 'lodash/fp'; -import { useFetchCurrentUserConversations } from '@kbn/elastic-assistant'; const BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY = unset( DATA_QUALITY_DASHBOARD_CONVERSATION_ID, @@ -49,7 +48,7 @@ jest.mock('../../common/lib/kibana', () => { }; }); -describe('useConversationStore', () => { +describe('useBaseConversations', () => { beforeEach(() => { jest.clearAllMocks(); @@ -58,56 +57,17 @@ describe('useConversationStore', () => { it('should return conversations with "Data Quality dashboard" conversation', () => { (useLinkAuthorized as jest.Mock).mockReturnValue(true); - const { result } = renderHook(() => useConversationStore()); + const { result } = renderHook(() => useBaseConversations()); expect(result.current).toEqual(expect.objectContaining(BASE_SECURITY_CONVERSATIONS)); }); it('should return conversations Without "Data Quality dashboard" conversation', () => { (useLinkAuthorized as jest.Mock).mockReturnValue(false); - const { result } = renderHook(() => useConversationStore()); + const { result } = renderHook(() => useBaseConversations()); expect(result.current).toEqual( expect.objectContaining(BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY) ); }); - - it('should return stored conversations merged with the base conversations', () => { - (useLinkAuthorized as jest.Mock).mockReturnValue(true); - - const persistedConversations = { - data: { - '1234': { - id: '1234', - title: 'Welcome', - isDefault: true, - messages: [], - apiConfig: { - connectorId: 'c29c28a0-20fe-11ee-9306-a1f4d42ec542', - provider: 'OpenAi', - }, - }, - '5657': { - id: '5657', - title: 'Test', - isDefault: true, - messages: [], - apiConfig: { - connectorId: 'c29c28a0-20fe-11ee-9306-a1f4d42ec542', - provider: 'OpenAi', - }, - }, - }, - isLoading: false, - isError: false, - }; - (useFetchCurrentUserConversations as jest.Mock).mockReturnValue(persistedConversations); - const { result } = renderHook(() => useConversationStore()); - - expect(result.current).toEqual( - expect.objectContaining(BASE_CONVERSATIONS_WITHOUT_DATA_QUALITY) - ); - - expect(result.current).toEqual(expect.objectContaining(persistedConversations.data)); - }); }); diff --git a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.ts b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.ts index 210fdd42c1555..4bfead43f1a36 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.ts @@ -72,6 +72,7 @@ export const OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL: AssistantTool = { localReplacements = { ...localReplacements, ...newReplacements }; // update the local state onNewReplacements?.(localReplacements); // invoke the callback with the latest replacements + return Promise.resolve(localReplacements); }; return JSON.stringify( From 2ea2fb7a89d35ea7e3e920216ecb20b65210f1a3 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 31 Jan 2024 20:25:47 -0800 Subject: [PATCH 060/141] Refactored logic with toast instead of silent error log --- .../api/conversations/conversations.test.tsx | 91 +++++++++++++++++++ .../api/conversations/conversations.ts | 84 ++++++++++++----- .../impl/assistant/use_conversation/index.tsx | 59 ++++-------- 3 files changed, 172 insertions(+), 62 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx new file mode 100644 index 0000000000000..4d7c8fe3a0d24 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act, renderHook } from '@testing-library/react-hooks'; + +import { + DeleteConversationParams, + GetConversationByIdParams, + deleteConversation, + getConversationById, +} from './conversations'; +import { HttpSetupMock } from '@kbn/core-http-browser-mocks'; +import { coreMock } from '@kbn/core/public/mocks'; + +let http: HttpSetupMock = coreMock.createSetup().http; + +const toasts = { + addError: jest.fn(), +}; + +describe('conversations api', () => { + beforeEach(() => { + jest.clearAllMocks(); + http = coreMock.createSetup().http; + }); + + it('should call api to delete conversation', async () => { + await act(async () => { + const deleteProps = { http, toasts, id: 'test' } as unknown as DeleteConversationParams; + + const { waitForNextUpdate } = renderHook(() => deleteConversation(deleteProps)); + await waitForNextUpdate(); + + expect(deleteProps.http.fetch).toHaveBeenCalledWith( + '/api/elastic_assistant/conversations/test', + { + method: 'DELETE', + signal: undefined, + version: '2023-10-31', + } + ); + expect(toasts.addError).not.toHaveBeenCalled(); + }); + }); + + it('should display error toast when delete api throws error', async () => { + http.fetch.mockRejectedValue(new Error('this is an error')); + const deleteProps = { http, toasts, id: 'test' } as unknown as DeleteConversationParams; + + await act(async () => { + const { waitForNextUpdate } = renderHook(() => deleteConversation(deleteProps)); + await waitForNextUpdate(); + + expect(toasts.addError).toHaveBeenCalled(); + }); + }); + + it('should call api to get conversation', async () => { + await act(async () => { + const getProps = { http, toasts, id: 'test' } as unknown as GetConversationByIdParams; + const { waitForNextUpdate } = renderHook(() => getConversationById(getProps)); + await waitForNextUpdate(); + + expect(getProps.http.fetch).toHaveBeenCalledWith( + '/api/elastic_assistant/conversations/test', + { + method: 'GET', + signal: undefined, + version: '2023-10-31', + } + ); + expect(toasts.addError).not.toHaveBeenCalled(); + }); + }); + + it('should display error toast when get api throws error', async () => { + http.fetch.mockRejectedValue(new Error('this is an error')); + const getProps = { http, toasts, id: 'test' } as unknown as GetConversationByIdParams; + + await act(async () => { + const { waitForNextUpdate } = renderHook(() => getConversationById(getProps)); + await waitForNextUpdate(); + + expect(toasts.addError).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts index cef370368e1d6..3589523b4ac6d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts @@ -6,8 +6,8 @@ */ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; -import { HttpSetup } from '@kbn/core/public'; -import { IHttpFetchError } from '@kbn/core-http-browser'; +import { HttpSetup, IToasts } from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, @@ -17,6 +17,7 @@ import { Conversation, Message } from '../../../assistant_context/types'; export interface GetConversationByIdParams { http: HttpSetup; id: string; + toasts?: IToasts; signal?: AbortSignal | undefined; } @@ -26,15 +27,17 @@ export interface GetConversationByIdParams { * @param {Object} options - The options object. * @param {HttpSetup} options.http - HttpSetup * @param {string} options.id - Conversation id. + * @param {IToasts} [options.toasts] - IToasts * @param {AbortSignal} [options.signal] - AbortSignal * - * @returns {Promise} + * @returns {Promise} */ export const getConversationById = async ({ http, id, signal, -}: GetConversationByIdParams): Promise => { + toasts, +}: GetConversationByIdParams): Promise => { try { const response = await http.fetch(`${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${id}`, { method: 'GET', @@ -44,13 +47,19 @@ export const getConversationById = async ({ return response as Conversation; } catch (error) { - return error as IHttpFetchError; + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.getConversationError', { + defaultMessage: 'Error fetching conversation by id {id}', + values: { id }, + }), + }); } }; export interface PostConversationParams { http: HttpSetup; conversation: Conversation; + toasts?: IToasts; signal?: AbortSignal | undefined; } @@ -61,14 +70,16 @@ export interface PostConversationParams { * @param {HttpSetup} options.http - HttpSetup * @param {Conversation} [options.conversation] - Conversation to be added * @param {AbortSignal} [options.signal] - AbortSignal + * @param {IToasts} [options.toasts] - IToasts * - * @returns {Promise} + * @returns {Promise} */ export const createConversation = async ({ http, conversation, signal, -}: PostConversationParams): Promise => { + toasts, +}: PostConversationParams): Promise => { try { const response = await http.post(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, { body: JSON.stringify(conversation), @@ -78,20 +89,22 @@ export const createConversation = async ({ return response as Conversation; } catch (error) { - return error as IHttpFetchError; + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.createConversationError', { + defaultMessage: 'Error creating conversation with title {title}', + values: { title: conversation.title }, + }), + }); } }; export interface DeleteConversationParams { http: HttpSetup; id: string; + toasts?: IToasts; signal?: AbortSignal | undefined; } -export interface DeleteConversationResponse { - success: boolean; -} - /** * API call for deleting the Conversation. Provide a id to delete that specific resource. * @@ -99,14 +112,16 @@ export interface DeleteConversationResponse { * @param {HttpSetup} options.http - HttpSetup * @param {string} [options.id] - Conversation id to be deleted * @param {AbortSignal} [options.signal] - AbortSignal + * @param {IToasts} [options.toasts] - IToasts * - * @returns {Promise} + * @returns {Promise} */ export const deleteConversation = async ({ http, id, signal, -}: DeleteConversationParams): Promise => { + toasts, +}: DeleteConversationParams): Promise => { try { const response = await http.fetch(`${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${id}`, { method: 'DELETE', @@ -114,14 +129,20 @@ export const deleteConversation = async ({ signal, }); - return response as DeleteConversationResponse; + return response as boolean; } catch (error) { - return error as IHttpFetchError; + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.deleteConversationError', { + defaultMessage: 'Error deleting conversation by id {id}', + values: { id }, + }), + }); } }; export interface PutConversationMessageParams { http: HttpSetup; + toasts?: IToasts; conversationId: string; title?: string; messages?: Message[]; @@ -133,6 +154,7 @@ export interface PutConversationMessageParams { model?: string; }; replacements?: Record; + excludeFromLastConversationStorage?: boolean; signal?: AbortSignal | undefined; } @@ -142,19 +164,25 @@ export interface PutConversationMessageParams { * @param {PutConversationMessageParams} options - The options object. * @param {HttpSetup} options.http - HttpSetup * @param {string} [options.title] - Conversation title + * @param {boolean} [options.excludeFromLastConversationStorage] - Conversation excludeFromLastConversationStorage + * @param {ApiConfig} [options.apiConfig] - Conversation apiConfig + * @param {Message[]} [options.messages] - Conversation messages + * @param {IToasts} [options.toasts] - IToasts * @param {AbortSignal} [options.signal] - AbortSignal * - * @returns {Promise} + * @returns {Promise} */ export const updateConversation = async ({ http, + toasts, title, conversationId, messages, apiConfig, replacements, + excludeFromLastConversationStorage, signal, -}: PutConversationMessageParams): Promise => { +}: PutConversationMessageParams): Promise => { try { const response = await http.fetch( `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${conversationId}`, @@ -166,6 +194,7 @@ export const updateConversation = async ({ messages, replacements, apiConfig, + excludeFromLastConversationStorage, }), headers: { 'Content-Type': 'application/json', @@ -177,7 +206,12 @@ export const updateConversation = async ({ return response as Conversation; } catch (error) { - return error as IHttpFetchError; + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.updateConversationError', { + defaultMessage: 'Error updating conversation by id {conversationId}', + values: { conversationId }, + }), + }); } }; @@ -186,14 +220,15 @@ export const updateConversation = async ({ * * @param {PutConversationMessageParams} options - The options object. * - * @returns {Promise} + * @returns {Promise} */ export const appendConversationMessages = async ({ http, conversationId, messages, signal, -}: PutConversationMessageParams): Promise => { + toasts, +}: PutConversationMessageParams): Promise => { try { const response = await http.fetch( `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${conversationId}/messages`, @@ -212,6 +247,11 @@ export const appendConversationMessages = async ({ return response as Conversation; } catch (error) { - return error as IHttpFetchError; + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.updateConversationError', { + defaultMessage: 'Error updating conversation by id {conversationId}', + values: { conversationId }, + }), + }); } }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index c26274d5ca812..b258f706f8cd2 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -7,7 +7,6 @@ import { useCallback } from 'react'; -import { IHttpFetchError, isHttpFetchError } from '@kbn/core-http-browser'; import { useAssistantContext } from '../../assistant_context'; import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; @@ -69,7 +68,7 @@ interface UseConversation { setApiConfig: ({ conversation, apiConfig, - }: SetApiConfigProps) => Promise>; + }: SetApiConfigProps) => Promise; createConversation: (conversation: Conversation) => Promise; getConversation: (conversationId: string) => Promise; } @@ -84,11 +83,7 @@ export const useConversation = (): UseConversation => { const getConversation = useCallback( async (conversationId: string) => { - const currentConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(currentConversation)) { - return; - } - return currentConversation; + return getConversationById({ http, id: conversationId }); }, [http] ); @@ -100,9 +95,6 @@ export const useConversation = (): UseConversation => { async (conversationId: string) => { let messages: Message[] = []; const prevConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(prevConversation)) { - return; - } if (prevConversation != null) { messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); await updateConversation({ @@ -122,9 +114,6 @@ export const useConversation = (): UseConversation => { const amendMessage = useCallback( async ({ conversationId, content }: AmendMessageProps) => { const prevConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(prevConversation)) { - return; - } if (prevConversation != null) { const { messages } = prevConversation; const message = messages[messages.length - 1]; @@ -157,10 +146,7 @@ export const useConversation = (): UseConversation => { conversationId, messages: [message], }); - if (isHttpFetchError(res)) { - return; - } - return res.messages; + return res?.messages; }, [assistantTelemetry, isEnabledKnowledgeBase, isEnabledRAGAlerts, http] ); @@ -172,9 +158,6 @@ export const useConversation = (): UseConversation => { }: AppendReplacementsProps): Promise | undefined> => { let allReplacements = replacements; const prevConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(prevConversation)) { - return; - } if (prevConversation != null) { allReplacements = { ...prevConversation.replacements, @@ -196,23 +179,22 @@ export const useConversation = (): UseConversation => { const clearConversation = useCallback( async (conversationId: string) => { const prevConversation = await getConversationById({ http, id: conversationId }); - if (isHttpFetchError(prevConversation)) { - return; - } - const defaultSystemPromptId = getDefaultSystemPrompt({ - allSystemPrompts, - conversation: prevConversation, - })?.id; + if (prevConversation) { + const defaultSystemPromptId = getDefaultSystemPrompt({ + allSystemPrompts, + conversation: prevConversation, + })?.id; - await updateConversation({ - http, - conversationId, - apiConfig: { - defaultSystemPromptId, - }, - messages: [], - replacements: undefined, - }); + await updateConversation({ + http, + conversationId, + apiConfig: { + defaultSystemPromptId, + }, + messages: [], + replacements: undefined, + }); + } }, [allSystemPrompts, http] ); @@ -249,10 +231,7 @@ export const useConversation = (): UseConversation => { */ const createConversation = useCallback( async (conversation: Conversation): Promise => { - const response = await createConversationApi({ http, conversation }); - if (!isHttpFetchError(response)) { - return response; - } + return createConversationApi({ http, conversation }); }, [http] ); From 712b7505178e894015e852b6e73626a8dec5a73f Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 1 Feb 2024 16:45:10 -0800 Subject: [PATCH 061/141] fixed tests --- .../use_bulk_actions_conversations.ts | 2 +- .../system_prompt_settings.tsx | 110 +++++++++--------- .../connector_selector_inline.tsx | 3 +- .../connectorland/connector_setup/index.tsx | 3 +- 4 files changed, 55 insertions(+), 63 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index e19e24988b094..16d23964b507b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -48,7 +48,7 @@ export interface BulkActionResponse { attributes: BulkActionAttributes; } -interface ConversationUpdateParams { +export interface ConversationUpdateParams { id?: string; title?: string; messages?: Message[]; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx index 545956627de92..de672294aac07 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx @@ -27,7 +27,7 @@ import * as i18n from './translations'; import { ConversationMultiSelector } from './conversation_multi_selector/conversation_multi_selector'; import { SystemPromptSelector } from './system_prompt_selector/system_prompt_selector'; import { TEST_IDS } from '../../../constants'; -import { ConversationsBulkActions } from '../../../api'; +import { ConversationUpdateParams, ConversationsBulkActions } from '../../../api'; interface Props { conversationSettings: Record; @@ -104,6 +104,16 @@ export const SystemPromptSettings: React.FC = React.memo( const handleConversationSelectionChange = useCallback( (currentPromptConversations: Conversation[]) => { const currentPromptConversationIds = currentPromptConversations.map((convo) => convo.id); + const getDefaultSystemPromptId = (convo: Conversation) => + currentPromptConversationIds.includes(convo.id) + ? selectedSystemPrompt?.id + : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id + ? // remove the default System Prompt if it is assigned to a conversation + // but that conversation is not in the currentPromptConversationList + // This means conversation was removed in the current transaction + undefined + : // leave it as it is .. if that conversation was neither added nor removed. + convo.apiConfig.defaultSystemPromptId; if (selectedSystemPrompt != null) { setConversationSettings((prev) => @@ -119,15 +129,7 @@ export const SystemPromptSettings: React.FC = React.memo( ...convo, apiConfig: { ...convo.apiConfig, - defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) - ? selectedSystemPrompt?.id - : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id - ? // remove the default System Prompt if it is assigned to a conversation - // but that conversation is not in the currentPromptConversationList - // This means conversation was removed in the current transaction - undefined - : // leave it as it is .. if that conversation was neither added nor removed. - convo.apiConfig.defaultSystemPromptId, + defaultSystemPromptId: getDefaultSystemPromptId(convo), }, })) ) @@ -135,56 +137,48 @@ export const SystemPromptSettings: React.FC = React.memo( let updatedConversationsSettingsBulkActions = { ...conversationsSettingsBulkActions }; Object.values(conversationSettings).forEach((convo) => { - if (convo.id !== convo.title) { - updatedConversationsSettingsBulkActions = { - ...updatedConversationsSettingsBulkActions, - update: { - ...(updatedConversationsSettingsBulkActions.update ?? {}), - [convo.id]: { - ...(updatedConversationsSettingsBulkActions.update - ? updatedConversationsSettingsBulkActions.update[convo.id] ?? {} - : {}), - apiConfig: { - ...((updatedConversationsSettingsBulkActions.update - ? updatedConversationsSettingsBulkActions.update[convo.id] ?? {} - : {} - ).apiConfig ?? {}), - defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) - ? selectedSystemPrompt?.id - : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id - ? // remove the default System Prompt if it is assigned to a conversation - // but that conversation is not in the currentPromptConversationList - // This means conversation was removed in the current transaction - undefined - : // leave it as it is .. if that conversation was neither added nor removed. - convo.apiConfig.defaultSystemPromptId, + const getApiConfig = ( + operation: Record | undefined + ) => ({ + ...((operation ? operation[convo.id] ?? {} : {}).apiConfig ?? {}), + defaultSystemPromptId: getDefaultSystemPromptId(convo), + }); + const createOperation = + convo.id === convo.title + ? { + create: { + ...(updatedConversationsSettingsBulkActions.create ?? {}), + [convo.id]: { + ...convo, + apiConfig: { + ...convo.apiConfig, + defaultSystemPromptId: getDefaultSystemPromptId(convo), + }, + }, }, - }, - }, - }; - } else { - updatedConversationsSettingsBulkActions = { - ...updatedConversationsSettingsBulkActions, - create: { - ...(updatedConversationsSettingsBulkActions.create ?? {}), - [convo.id]: { - ...convo, - apiConfig: { - ...convo.apiConfig, - defaultSystemPromptId: currentPromptConversationIds.includes(convo.id) - ? selectedSystemPrompt?.id - : convo.apiConfig.defaultSystemPromptId === selectedSystemPrompt?.id - ? // remove the default System Prompt if it is assigned to a conversation - // but that conversation is not in the currentPromptConversationList - // This means conversation was removed in the current transaction - undefined - : // leave it as it is .. if that conversation was neither added nor removed. - convo.apiConfig.defaultSystemPromptId, + } + : {}; + + const updateOperation = + convo.id === convo.title + ? { + update: { + ...(updatedConversationsSettingsBulkActions.update ?? {}), + [convo.id]: { + ...(updatedConversationsSettingsBulkActions.update + ? updatedConversationsSettingsBulkActions.update[convo.id] ?? {} + : {}), + apiConfig: getApiConfig(updatedConversationsSettingsBulkActions.update), + }, }, - }, - }, - }; - } + } + : {}; + + updatedConversationsSettingsBulkActions = { + ...updatedConversationsSettingsBulkActions, + ...createOperation, + ...updateOperation, + }; }); setConversationsSettingsBulkActions(updatedConversationsSettingsBulkActions); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx index 4ea89ef252348..eb43f1ae6eac8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.tsx @@ -9,7 +9,6 @@ import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui import React, { useCallback, useMemo, useState } from 'react'; import { css } from '@emotion/css'; -import { isHttpFetchError } from '@kbn/core-http-browser'; import { AIConnector, ConnectorSelector } from '../connector_selector'; import { Conversation } from '../../..'; import { useLoadConnectors } from '../use_load_connectors'; @@ -119,7 +118,7 @@ export const ConnectorSelectorInline: React.FC = React.memo( }, }); - if (!isHttpFetchError(conversation)) { + if (conversation) { onConnectorSelected(conversation); } } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 82fe0a2e98fb7..de0ed6374fe98 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -13,7 +13,6 @@ import styled from 'styled-components'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public/common/constants'; import { ActionType } from '@kbn/triggers-actions-ui-plugin/public'; -import { isHttpFetchError } from '@kbn/core-http-browser'; import { AddConnectorModal } from '../add_connector_modal'; import { WELCOME_CONVERSATION } from '../../assistant/use_conversation/sample_conversations'; import { Conversation, Message } from '../../..'; @@ -189,7 +188,7 @@ export const useConnectorSetup = ({ }, }); - if (!isHttpFetchError(updatedConversation)) { + if (updatedConversation) { onConversationUpdate({ cId: updatedConversation.id, cTitle: updatedConversation.title }); refetchConnectors?.(); From c4545f5ce48e73c0449f4ac4fbabe18060a3b5c6 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 1 Feb 2024 17:53:50 -0800 Subject: [PATCH 062/141] moved schemas to common package --- ...ost_actions_connector_execute_route.gen.ts | 0 ...ctions_connector_execute_route.schema.yaml | 0 ...lk_crud_anonymization_fields_route.gen.ts} | 0 ...ud_anonymization_fields_route.schema.yaml} | 0 .../find_anonymization_fields_route.gen.ts} | 2 +- ...nd_anonymization_fields_route.schema.yaml} | 0 .../impl}/schemas/common.ts | 0 .../bulk_crud_conversations_route.gen.ts | 0 .../bulk_crud_conversations_route.schema.yaml | 0 .../conversations/common_attributes.gen.ts | 0 .../common_attributes.schema.yaml | 0 .../crud_conversation_route.gen.ts | 0 .../crud_conversation_route.schema.yaml | 0 .../find_conversations_route.gen.ts | 0 .../find_conversations_route.schema.yaml | 0 .../impl/schemas/index.ts | 15 ++++ .../knowledge_base/crud_kb_route.gen.ts | 0 .../knowledge_base/crud_kb_route.schema.yaml | 0 .../schemas/prompts/crud_prompts_route.gen.ts | 0 .../prompts/crud_prompts_route.schema.yaml | 0 .../schemas/prompts/find_prompts_route.gen.ts | 0 .../prompts/find_prompts_route.schema.yaml | 0 .../__mocks__/conversations_schema.mock.ts | 6 +- .../server/__mocks__/request.ts | 6 +- .../server/__mocks__/response.ts | 2 +- .../append_conversation_messages.ts | 6 +- .../conversations_data_writer.ts | 2 +- .../create_conversation.test.ts | 5 +- .../create_conversation.ts | 2 +- .../delete_conversation.test.ts | 2 +- .../delete_conversation.ts | 2 +- .../find_conversations.ts | 2 +- .../get_conversation.test.ts | 2 +- .../get_conversation.ts | 2 +- .../server/conversations_data_client/index.ts | 12 +-- .../conversations_data_client/transforms.ts | 2 +- .../server/conversations_data_client/types.ts | 7 +- .../update_conversation.test.ts | 5 +- .../update_conversation.ts | 2 +- .../server/lib/executor.test.ts | 2 +- .../elastic_assistant/server/lib/executor.ts | 2 +- .../server/lib/langchain/executors/types.ts | 2 +- .../server/lib/langchain/helpers.test.ts | 2 +- .../server/lib/langchain/helpers.ts | 2 +- .../langchain/llm/actions_client_llm.test.ts | 2 +- .../lib/langchain/llm/actions_client_llm.ts | 2 +- .../bulk_actions_route.ts | 10 +-- .../routes/anonimization_fields/find_route.ts | 9 ++- .../anonimization_fields/update_route.ts | 79 +++++++++++++++++++ .../capabilities/get_capabilities_route.ts | 2 +- .../append_conversation_messages_route.ts | 8 +- .../conversations/bulk_actions_route.ts | 15 ++-- .../routes/conversations/create_route.ts | 6 +- .../routes/conversations/delete_route.ts | 2 +- .../server/routes/conversations/find_route.ts | 6 +- .../find_user_conversations_route.ts | 4 +- .../server/routes/conversations/read_route.ts | 4 +- .../routes/conversations/update_route.ts | 8 +- .../server/routes/evaluate/get_evaluate.ts | 2 +- .../server/routes/evaluate/post_evaluate.ts | 4 +- .../server/routes/helpers.test.ts | 2 +- .../knowledge_base/delete_knowledge_base.ts | 8 +- .../routes/knowledge_base/get_kb_resource.ts | 2 +- .../get_knowledge_base_status.ts | 10 +-- .../knowledge_base/post_knowledge_base.ts | 10 +-- .../routes/post_actions_connector_execute.ts | 8 +- .../server/routes/prompts/create_route.ts | 3 +- .../server/routes/prompts/find_route.ts | 8 +- .../server/routes/prompts/update_route.ts | 5 +- ...ssistant_anonimization_fields_so_client.ts | 18 ++--- .../ai_assistant_prompts_so_client.ts | 13 +-- ...tic_assistant_anonimization_fields_type.ts | 4 +- .../elastic_assistant_prompts_type.ts | 4 +- .../plugins/elastic_assistant/server/types.ts | 3 +- .../alert_counts/alert_counts_tool.test.ts | 2 +- .../esql_language_knowledge_base_tool.test.ts | 2 +- .../open_and_acknowledged_alerts_tool.test.ts | 2 +- 77 files changed, 218 insertions(+), 143 deletions(-) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/actions_connector/post_actions_connector_execute_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts => packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts} (100%) rename x-pack/{plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml => packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml} (100%) rename x-pack/{plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts => packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts} (97%) rename x-pack/{plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml => packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml} (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/common.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/bulk_crud_conversations_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/bulk_crud_conversations_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/common_attributes.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/common_attributes.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/crud_conversation_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/crud_conversation_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/find_conversations_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/conversations/find_conversations_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/knowledge_base/crud_kb_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/knowledge_base/crud_kb_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/prompts/crud_prompts_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/prompts/crud_prompts_route.schema.yaml (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/prompts/find_prompts_route.gen.ts (100%) rename x-pack/{plugins/elastic_assistant/server => packages/kbn-elastic-assistant-common/impl}/schemas/prompts/find_prompts_route.schema.yaml (100%) create mode 100644 x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/bulk_crud_anonimization_fields_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts similarity index 97% rename from x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts index 4ffb41d39e359..04f48d117d658 100644 --- a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts @@ -17,7 +17,7 @@ import { ArrayFromString } from '@kbn/zod-helpers'; * version: 2023-10-31 */ -import { AnonimizationFieldResponse } from './bulk_crud_anonimization_fields_route.gen'; +import { AnonimizationFieldResponse } from './bulk_crud_anonymization_fields_route.gen'; export type FindAnonimizationFieldsSortField = z.infer; export const FindAnonimizationFieldsSortField = z.enum([ diff --git a/x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/anonimization_fields/find_prompts_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/common.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/common.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/common.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/common.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/bulk_crud_conversations_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/bulk_crud_conversations_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/bulk_crud_conversations_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/bulk_crud_conversations_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/bulk_crud_conversations_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/common_attributes.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/crud_conversation_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/find_conversations_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/find_conversations_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/find_conversations_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/conversations/find_conversations_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/find_conversations_route.schema.yaml diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts index 4257cb9bae149..e898e87ab3ff8 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts @@ -24,3 +24,18 @@ export * from './evaluation/get_evaluate_route.gen'; // Capabilities Schemas export * from './capabilities/get_capabilities_route.gen'; + +// Conversations Schemas +export * from './conversations/bulk_crud_conversations_route.gen'; +export * from './conversations/common_attributes.gen'; +export * from './conversations/crud_conversation_route.gen'; +export * from './conversations/find_conversations_route.gen'; + +// Actions Connector Schemas +export * from './actions_connector/post_actions_connector_execute_route.gen'; + +// KB Schemas +export * from './knowledge_base/crud_kb_route.gen'; + +// Prompts Schemas +export * from './prompts/crud_prompts_route.gen'; diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/knowledge_base/crud_kb_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/prompts/crud_prompts_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.gen.ts rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts diff --git a/x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml similarity index 100% rename from x-pack/plugins/elastic_assistant/server/schemas/prompts/find_prompts_route.schema.yaml rename to x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 47d9cb4908207..7e194487426df 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -5,13 +5,13 @@ * 2.0. */ -import { PerformBulkActionRequestBody } from '../schemas/conversations/bulk_crud_conversations_route.gen'; import { + AppendConversationMessageRequestBody, + PerformBulkActionRequestBody, ConversationCreateProps, ConversationResponse, ConversationUpdateProps, -} from '../schemas/conversations/common_attributes.gen'; -import { AppendConversationMessageRequestBody } from '../schemas/conversations/crud_conversation_route.gen'; +} from '@kbn/elastic-assistant-common'; export const getCreateConversationSchemaMock = (): ConversationCreateProps => ({ title: 'Welcome', diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index dab6b37c51476..ad0370207e404 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -7,6 +7,8 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants'; import { + ConversationCreateProps, + ConversationUpdateProps, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, @@ -21,10 +23,6 @@ import { getCreateConversationSchemaMock, getUpdateConversationSchemaMock, } from './conversations_schema.mock'; -import { - ConversationCreateProps, - ConversationUpdateProps, -} from '../schemas/conversations/common_attributes.gen'; export const requestMock = { create: httpServerMock.createKibanaRequest, diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts index b18485feaf01b..d64ef1dc410c4 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/response.ts @@ -8,7 +8,7 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { getConversationMock, getQueryConversationParams } from './conversations_schema.mock'; import { estypes } from '@elastic/elasticsearch'; -import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; +import { ConversationResponse } from '@kbn/elastic-assistant-common'; export const responseMock = { create: httpServerMock.createResponseFactory, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index d2fbb51303f9b..76a9360fbb1db 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -7,11 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { - ConversationResponse, - Message, - UUID, -} from '../schemas/conversations/common_attributes.gen'; +import { ConversationResponse, Message, UUID } from '@kbn/elastic-assistant-common'; import { getConversation } from './get_conversation'; export interface AppendConversationMessagesParams { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index 54f99e3d5059a..28d5bfcce398a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -12,7 +12,7 @@ import { ConversationCreateProps, ConversationUpdateProps, UUID, -} from '../schemas/conversations/common_attributes.gen'; +} from '@kbn/elastic-assistant-common'; import { transformToCreateScheme } from './create_conversation'; import { transformToUpdateScheme } from './update_conversation'; import { SearchEsConversationSchema } from './types'; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index e0604d8d10775..0cfda1c5fd8c8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -7,14 +7,11 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { createConversation } from './create_conversation'; -import { - ConversationCreateProps, - ConversationResponse, -} from '../schemas/conversations/common_attributes.gen'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { estypes } from '@elastic/elasticsearch'; import { SearchEsConversationSchema } from './types'; import { getConversation } from './get_conversation'; +import { ConversationCreateProps, ConversationResponse } from '@kbn/elastic-assistant-common'; jest.mock('./get_conversation', () => ({ getConversation: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 1920874c90d64..ede06f3462e88 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -16,7 +16,7 @@ import { Reader, Replacement, UUID, -} from '../schemas/conversations/common_attributes.gen'; +} from '@kbn/elastic-assistant-common'; import { getConversation } from './get_conversation'; export interface CreateMessageSchema { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index 1bf20217aaebd..cee596c69cc02 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -8,8 +8,8 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { DeleteConversationParams, deleteConversation } from './delete_conversation'; import { getConversation } from './get_conversation'; -import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { ConversationResponse } from '@kbn/elastic-assistant-common'; jest.mock('./get_conversation', () => ({ getConversation: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index 06079def47947..440250d29f41b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -6,8 +6,8 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { UUID } from '@kbn/elastic-assistant-common'; import { getConversation } from './get_conversation'; -import { UUID } from '../schemas/conversations/common_attributes.gen'; export interface DeleteConversationParams { esClient: ElasticsearchClient; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts index 6f62082643d80..8d1a4f9fc5f5e 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/find_conversations.ts @@ -10,9 +10,9 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { estypes } from '@elastic/elasticsearch'; import { EsQueryConfig, Query, buildEsQuery } from '@kbn/es-query'; +import { FindConversationsResponse } from '@kbn/elastic-assistant-common'; import { transformESToConversations } from './transforms'; import { SearchEsConversationSchema } from './types'; -import { FindConversationsResponse } from '../schemas/conversations/find_conversations_route.gen'; interface FindConversationsOptions { filter?: string; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index 5820307ffa0d1..ee01db128dec6 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -10,8 +10,8 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m import { getConversation } from './get_conversation'; import { estypes } from '@elastic/elasticsearch'; import { SearchEsConversationSchema } from './types'; -import { ConversationResponse } from '../schemas/conversations/common_attributes.gen'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { ConversationResponse } from '@kbn/elastic-assistant-common'; export const getConversationResponseMock = (): ConversationResponse => ({ createdAt: '2020-04-20T15:25:31.830Z', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts index 3cc0244b5419c..7f694b27a9741 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { ConversationResponse, UUID } from '../schemas/conversations/common_attributes.gen'; +import { ConversationResponse, UUID } from '@kbn/elastic-assistant-common'; import { SearchEsConversationSchema } from './types'; import { transformESToConversations } from './transforms'; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index dd2bbcf869dab..a4355100af163 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -11,17 +11,17 @@ import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { AuthenticatedUser } from '@kbn/security-plugin/server'; import { estypes } from '@elastic/elasticsearch'; -import { IIndexPatternString } from '../types'; -import { ConversationDataWriter } from './conversations_data_writer'; -import { getIndexTemplateAndPattern } from '../ai_assistant_service/conversation_configuration_type'; -import { createConversation } from './create_conversation'; import { ConversationCreateProps, ConversationResponse, ConversationUpdateProps, + FindConversationsResponse, Message, -} from '../schemas/conversations/common_attributes.gen'; -import { FindConversationsResponse } from '../schemas/conversations/find_conversations_route.gen'; +} from '@kbn/elastic-assistant-common'; +import { IIndexPatternString } from '../types'; +import { ConversationDataWriter } from './conversations_data_writer'; +import { getIndexTemplateAndPattern } from '../ai_assistant_service/conversation_configuration_type'; +import { createConversation } from './create_conversation'; import { findConversations } from './find_conversations'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index 9e2eab800f9ec..8848c2efe9fbc 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -6,8 +6,8 @@ */ import { estypes } from '@elastic/elasticsearch'; +import { ConversationResponse, Replacement } from '@kbn/elastic-assistant-common'; import { SearchEsConversationSchema } from './types'; -import { ConversationResponse, Replacement } from '../schemas/conversations/common_attributes.gen'; export const transformESToConversations = ( response: estypes.SearchResponse diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts index 8ecc8e0568537..6e0fcc01aa2bd 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts @@ -5,12 +5,7 @@ * 2.0. */ -import { - MessageRole, - Provider, - Reader, - Replacement, -} from '../schemas/conversations/common_attributes.gen'; +import { MessageRole, Provider, Reader, Replacement } from '@kbn/elastic-assistant-common'; export interface SearchEsConversationSchema { id: string; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 221dda2ccc435..50208e2c17b55 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -9,10 +9,7 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m import { loggerMock } from '@kbn/logging-mocks'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; -import { - ConversationResponse, - ConversationUpdateProps, -} from '../schemas/conversations/common_attributes.gen'; +import { ConversationResponse, ConversationUpdateProps } from '@kbn/elastic-assistant-common'; export const getUpdateConversationOptionsMock = (): ConversationUpdateProps => ({ id: 'test', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 2b734e9cf3e5a..8b5cd8a46211f 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -14,7 +14,7 @@ import { UUID, Provider, MessageRole, -} from '../schemas/conversations/common_attributes.gen'; +} from '@kbn/elastic-assistant-common'; import { getConversation } from './get_conversation'; export interface UpdateConversationSchema { diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts index 11aab9d753335..83b0578d8a77b 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts @@ -15,7 +15,7 @@ import { executeAction, Props } from './executor'; import { PassThrough } from 'stream'; import { KibanaRequest } from '@kbn/core-http-server'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; -import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; const request = { body: { params: {}, diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index 9e832ee90edff..5ccab3513ce16 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -9,7 +9,7 @@ import { get } from 'lodash/fp'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { KibanaRequest } from '@kbn/core-http-server'; import { PassThrough, Readable } from 'stream'; -import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; export interface Props { actions: ActionsPluginStart; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts index 3fa3b207fcc23..08bdfe12e8058 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts @@ -12,7 +12,7 @@ import { Logger } from '@kbn/logging'; import { KibanaRequest } from '@kbn/core-http-server'; import type { LangChainTracer } from 'langchain/callbacks'; import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server'; -import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; import { ResponseBody } from '../types'; import type { AssistantTool } from '../../../types'; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts index 32f226f340e16..1584084eee0fb 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.test.ts @@ -8,6 +8,7 @@ import { KibanaRequest } from '@kbn/core-http-server'; import type { Message } from '@kbn/elastic-assistant'; import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from 'langchain/schema'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; import { getLangChainMessage, @@ -16,7 +17,6 @@ import { requestHasRequiredAnonymizationParams, } from './helpers'; import { langChainMessages } from '../../__mocks__/lang_chain_messages'; -import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('helpers', () => { describe('getLangChainMessage', () => { diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts index deb11bd9cf609..cb03c2b1deeda 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts @@ -9,7 +9,7 @@ import { KibanaRequest } from '@kbn/core-http-server'; import type { Message } from '@kbn/elastic-assistant'; import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from 'langchain/schema'; -import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen'; export const getLangChainMessage = ( assistantMessage: Pick diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts index ad36eddfc0b6f..70d238c0e7445 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.test.ts @@ -11,7 +11,7 @@ import { loggerMock } from '@kbn/logging-mocks'; import { ActionsClientLlm } from './actions_client_llm'; import { mockActionResponse } from '../../../__mocks__/action_result_data'; -import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; const connectorId = 'mock-connector-id'; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts index 9e504c2c98221..aa5a29482209a 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts @@ -11,7 +11,7 @@ import type { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plu import { LLM } from 'langchain/llms/base'; import { get } from 'lodash/fp'; -import { ExecuteConnectorRequestBody } from '../../../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; import { getMessageContentAndRole } from '../helpers'; const LLM_TYPE = 'ActionsClientLlm'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts index 8c8f59ff612ee..062dd7bd3a199 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts @@ -15,10 +15,6 @@ import { } from '@kbn/elastic-assistant-common'; import { SavedObjectError } from '@kbn/core/types'; -import { ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; import { AnonimizationFieldResponse, BulkActionSkipResult, @@ -27,7 +23,11 @@ import { BulkCrudActionSummary, PerformBulkActionRequestBody, PerformBulkActionResponse, -} from '../../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; +import { ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; export interface BulkOperationError { message: string; diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts index 1d6e7025c11de..2ad72df91b362 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts @@ -12,13 +12,14 @@ import { ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; + import { FindAnonimizationFieldsRequestQuery, FindAnonimizationFieldsResponse, -} from '../../schemas/anonimization_fields/find_prompts_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; export const findAnonimizationFieldsRoute = ( router: ElasticAssistantPluginRouter, diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts new file mode 100644 index 0000000000000..45dd9bf5c09ae --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { schema } from '@kbn/config-schema'; +import { + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, +} from '@kbn/elastic-assistant-common'; +import { + PromptResponse, + PromptUpdateProps, +} from '@kbn/elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; + +export const updatePromptRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .put({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PromptUpdateProps), + params: schema.object({ + promptId: schema.string(), + }), + }, + }, + }, + async (context, request, response): Promise> => { + const assistantResponse = buildResponse(response); + const { promptId } = request.params; + + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + + const existingPrompt = await dataClient?.getPrompt(promptId); + if (existingPrompt == null) { + return assistantResponse.error({ + body: `Prompt id: "${promptId}" not found`, + statusCode: 404, + }); + } + const prompt = await dataClient?.updatePromptItem(existingPrompt, request.body); + if (prompt == null) { + return assistantResponse.error({ + body: `prompt id: "${promptId}" was not updated`, + statusCode: 400, + }); + } + return response.ok({ + body: prompt, + }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts index 7c470cdfc2d94..1cd52553b5e47 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts @@ -13,12 +13,12 @@ import { GetCapabilitiesResponse, INTERNAL_API_ACCESS, } from '@kbn/elastic-assistant-common'; +import { buildRouteValidationWithZod } from '@kbn/elastic-assistant-common/impl/schemas/common'; import { CAPABILITIES } from '../../../common/constants'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { buildResponse } from '../../lib/build_response'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -import { buildRouteValidationWithZod } from '../../schemas/common'; /** * Get the assistant capabilities for the requesting plugin diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts index d22fc44743384..74416f91c4f64 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts @@ -8,17 +8,15 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { + ConversationResponse, + AppendConversationMessageRequestBody, + AppendConversationMessageRequestParams, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, } from '@kbn/elastic-assistant-common'; -import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; -import { - AppendConversationMessageRequestBody, - AppendConversationMessageRequestParams, -} from '../../schemas/conversations/crud_conversation_route.gen'; export const appendConversationMessageRoute = (router: ElasticAssistantPluginRouter) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts index e7e61244ec5a1..39a48b1737a81 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts @@ -12,21 +12,18 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, -} from '@kbn/elastic-assistant-common'; - -import { CONVERSATIONS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; -import { BulkActionSkipResult, BulkCrudActionResponse, BulkCrudActionResults, BulkCrudActionSummary, PerformBulkActionRequestBody, PerformBulkActionResponse, -} from '../../schemas/conversations/bulk_crud_conversations_route.gen'; -import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; + ConversationResponse, +} from '@kbn/elastic-assistant-common'; +import { CONVERSATIONS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; export interface BulkOperationError { message: string; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts index 87bf34e80a425..d987d90371cb9 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts @@ -10,12 +10,10 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL, -} from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { ConversationCreateProps, ConversationResponse, -} from '../../schemas/conversations/common_attributes.gen'; +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; import { buildResponse } from '../utils'; import { buildRouteValidationWithZod } from '../route_validation'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts index 7e85d24bc8107..77f509b3e82a5 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts @@ -9,10 +9,10 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, + DeleteConversationRequestParams, } from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildResponse } from '../utils'; -import { DeleteConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; import { buildRouteValidationWithZod } from '../route_validation'; export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts index a3757915adc24..11ab3005e987e 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts @@ -11,12 +11,10 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, -} from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { FindConversationsRequestQuery, FindConversationsResponse, -} from '../../schemas/conversations/find_conversations_route.gen'; +} from '@kbn/elastic-assistant-common'; +import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts index c8e1353caf4fc..2a3e34ae64aef 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts @@ -12,11 +12,11 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, } from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; import { FindConversationsRequestQuery, FindConversationsResponse, -} from '../../schemas/conversations/find_conversations_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/conversations/find_conversations_route.gen'; +import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts index a6701f7c35ea7..c7e2f4850cd5c 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts @@ -11,11 +11,11 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, } from '@kbn/elastic-assistant-common'; -import { ConversationResponse } from '../../schemas/conversations/common_attributes.gen'; +import { ConversationResponse } from '@kbn/elastic-assistant-common/impl/schemas/conversations/common_attributes.gen'; +import { ReadConversationRequestParams } from '@kbn/elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.gen'; import { buildResponse } from '../utils'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; -import { ReadConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts index 6cdb50724b4a7..aa64d96dc3aae 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts @@ -11,14 +11,14 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, } from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; import { ConversationResponse, ConversationUpdateProps, -} from '../../schemas/conversations/common_attributes.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/conversations/common_attributes.gen'; +import { UpdateConversationRequestParams } from '@kbn/elastic-assistant-common/impl/schemas/conversations/crud_conversation_route.gen'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; -import { UpdateConversationRequestParams } from '../../schemas/conversations/crud_conversation_route.gen'; export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts index bc9922ef5f35a..ea577dcf235b5 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts @@ -17,8 +17,8 @@ import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { EVALUATE } from '../../../common/constants'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; -import { buildRouteValidationWithZod } from '../../schemas/common'; import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; +import { buildRouteValidationWithZod } from '../route_validation'; export const getEvaluateRoute = (router: IRouter) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index 33ada7d338b05..f43985187e238 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -15,7 +15,9 @@ import { PostEvaluateBody, PostEvaluateRequestQuery, PostEvaluateResponse, + ExecuteConnectorRequestBody, } from '@kbn/elastic-assistant-common'; +import { buildRouteValidationWithZod } from '@kbn/elastic-assistant-common/impl/schemas/common'; import { ESQL_RESOURCE } from '../knowledge_base/constants'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; @@ -34,9 +36,7 @@ import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers'; * To support additional Agent Executors from the UI, add them to this map * and reference your specific AgentExecutor function */ -import { buildRouteValidationWithZod } from '../../schemas/common'; import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors'; -import { ExecuteConnectorRequestBody } from '../../schemas/actions_connector/post_actions_connector_execute_route.gen'; const DEFAULT_SIZE = 20; diff --git a/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts b/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts index f298b4a17e0a2..2ac00be311b9b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/helpers.test.ts @@ -8,7 +8,7 @@ import type { KibanaRequest } from '@kbn/core-http-server'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from './helpers'; -import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; +import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; describe('getPluginNameFromRequest', () => { const contextRequestHeaderEncoded = encodeURIComponent( diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts index c3cb647c2996c..e298284f4d344 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts @@ -9,16 +9,16 @@ import { IRouter, KibanaRequest } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { + DeleteKnowledgeBaseRequestParams, + DeleteKnowledgeBaseResponse, +} from '@kbn/elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantRequestHandlerContext } from '../../types'; import { KNOWLEDGE_BASE } from '../../../common/constants'; import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elasticsearch_store'; import { ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; import { buildRouteValidationWithZod } from '../route_validation'; -import { - DeleteKnowledgeBaseRequestParams, - DeleteKnowledgeBaseResponse, -} from '../../schemas/knowledge_base/crud_kb_route.gen'; /** * Delete Knowledge Base index, pipeline, and resources (collection of documents) diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts index df3e112e1704c..9dc0edf8e482d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_kb_resource.ts @@ -6,7 +6,7 @@ */ import { KibanaRequest } from '@kbn/core/server'; -import { CreateKnowledgeBaseRequestParams } from '../../schemas/knowledge_base/crud_kb_route.gen'; +import { CreateKnowledgeBaseRequestParams } from '@kbn/elastic-assistant-common'; /** * Returns the optional resource, e.g. `esql` from the request params, or undefined if it doesn't exist diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts index cffbf3836c1fa..2059d3af7aad8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts @@ -7,7 +7,11 @@ import { transformError } from '@kbn/securitysolution-es-utils'; -import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { + ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + ReadKnowledgeBaseRequestParams, + ReadKnowledgeBaseResponse, +} from '@kbn/elastic-assistant-common'; import { KibanaRequest } from '@kbn/core/server'; import { getKbResource } from './get_kb_resource'; import { buildResponse } from '../../lib/build_response'; @@ -15,10 +19,6 @@ import { ElasticAssistantPluginRouter, GetElser } from '../../types'; import { KNOWLEDGE_BASE } from '../../../common/constants'; import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elasticsearch_store'; import { ESQL_DOCS_LOADED_QUERY, ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; -import { - ReadKnowledgeBaseRequestParams, - ReadKnowledgeBaseResponse, -} from '../../schemas/knowledge_base/crud_kb_route.gen'; import { buildRouteValidationWithZod } from '../route_validation'; /** diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts index 868cd72c42aef..77c2fbe26b260 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts @@ -7,7 +7,11 @@ import { transformError } from '@kbn/securitysolution-es-utils'; -import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { + ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + CreateKnowledgeBaseRequestParams, + CreateKnowledgeBaseResponse, +} from '@kbn/elastic-assistant-common'; import { IKibanaResponse, KibanaRequest } from '@kbn/core/server'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantPluginRouter, GetElser } from '../../types'; @@ -16,10 +20,6 @@ import { ElasticsearchStore } from '../../lib/langchain/elasticsearch_store/elas import { ESQL_DOCS_LOADED_QUERY, ESQL_RESOURCE, KNOWLEDGE_BASE_INDEX_PATTERN } from './constants'; import { getKbResource } from './get_kb_resource'; import { loadESQL } from '../../lib/langchain/content_loaders/esql_loader'; -import { - CreateKnowledgeBaseRequestParams, - CreateKnowledgeBaseResponse, -} from '../../schemas/knowledge_base/crud_kb_route.gen'; import { buildRouteValidationWithZod } from '../route_validation'; /** diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index fe8b48435d77d..6c60c6c3cc698 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -9,7 +9,11 @@ import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { schema } from '@kbn/config-schema'; -import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION } from '@kbn/elastic-assistant-common'; +import { + ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + ExecuteConnectorRequestBody, + Message, +} from '@kbn/elastic-assistant-common'; import { INVOKE_ASSISTANT_ERROR_EVENT, INVOKE_ASSISTANT_SUCCESS_EVENT, @@ -25,9 +29,7 @@ import { ElasticAssistantRequestHandlerContext, GetElser } from '../types'; import { ESQL_RESOURCE } from './knowledge_base/constants'; import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain'; import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from './helpers'; -import { ExecuteConnectorRequestBody } from '../schemas/actions_connector/post_actions_connector_execute_route.gen'; import { buildRouteValidationWithZod } from './route_validation'; -import { Message } from '../schemas/conversations/common_attributes.gen'; export const postActionsConnectorExecuteRoute = ( router: IRouter, diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts index 8b4c74b3f295d..ea776bc7c2d69 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts @@ -8,13 +8,14 @@ import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { + PromptCreateProps, + PromptResponse, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_PROMPTS_URL, } from '@kbn/elastic-assistant-common'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildResponse } from '../utils'; import { buildRouteValidationWithZod } from '../route_validation'; -import { PromptCreateProps, PromptResponse } from '../../schemas/prompts/crud_prompts_route.gen'; export const createPromptRoute = (router: ElasticAssistantPluginRouter): void => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts index 00d60a95d6900..8eb7c2f291f52 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts @@ -12,13 +12,13 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND, } from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; import { FindPromptsRequestQuery, FindPromptsResponse, -} from '../../schemas/prompts/find_prompts_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; export const findPromptsRoute = (router: ElasticAssistantPluginRouter, logger: Logger) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts index 740e990d06583..45dd9bf5c09ae 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts @@ -12,10 +12,13 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, } from '@kbn/elastic-assistant-common'; +import { + PromptResponse, + PromptUpdateProps, +} from '@kbn/elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; -import { PromptResponse, PromptUpdateProps } from '../../schemas/prompts/crud_prompts_route.gen'; export const updatePromptRoute = (router: ElasticAssistantPluginRouter) => { router.versioned diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts index 4584c2179398e..fa588f118539d 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts @@ -12,22 +12,22 @@ import { SavedObjectsBulkDeleteStatus, } from '@kbn/core/server'; -import { - AssistantAnonimizationFieldSoSchema, - assistantAnonimizationFieldsTypeName, - transformSavedObjectToAssistantAnonimizationField, - transformSavedObjectUpdateToAssistantAnonimizationField, - transformSavedObjectsToFoundAssistantAnonimizationField, -} from './elastic_assistant_anonimization_fields_type'; import { AnonimizationFieldCreateProps, AnonimizationFieldResponse, AnonimizationFieldUpdateProps, -} from '../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; import { FindAnonimizationFieldsResponse, SortOrder, -} from '../schemas/anonimization_fields/find_prompts_route.gen'; +} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; +import { + AssistantAnonimizationFieldSoSchema, + assistantAnonimizationFieldsTypeName, + transformSavedObjectToAssistantAnonimizationField, + transformSavedObjectUpdateToAssistantAnonimizationField, + transformSavedObjectsToFoundAssistantAnonimizationField, +} from './elastic_assistant_anonimization_fields_type'; export interface ConstructorOptions { /** User creating, modifying, deleting, or updating the anonimization fields */ diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts index f41b347eb551c..0820fba23e7ba 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts @@ -11,6 +11,13 @@ import { type SavedObjectsClientContract, } from '@kbn/core/server'; +import { + PromptCreateProps, + PromptResponse, + PromptUpdateProps, + SortOrder, +} from '@kbn/elastic-assistant-common'; +import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; import { AssistantPromptSoSchema, assistantPromptsTypeName, @@ -18,12 +25,6 @@ import { transformSavedObjectUpdateToAssistantPrompt, transformSavedObjectsToFoundAssistantPrompt, } from './elastic_assistant_prompts_type'; -import { - PromptCreateProps, - PromptResponse, - PromptUpdateProps, -} from '../schemas/prompts/crud_prompts_route.gen'; -import { FindPromptsResponse, SortOrder } from '../schemas/prompts/find_prompts_route.gen'; export interface ConstructorOptions { /** User creating, modifying, deleting, or updating the prompts */ diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts index c61b726cea8a3..b3aa65355bba0 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts @@ -12,8 +12,8 @@ import type { SavedObjectsType, SavedObjectsUpdateResponse, } from '@kbn/core/server'; -import { AnonimizationFieldResponse } from '../schemas/anonimization_fields/bulk_crud_anonimization_fields_route.gen'; -import { FindAnonimizationFieldsResponse } from '../schemas/anonimization_fields/find_prompts_route.gen'; +import { AnonimizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; +import { FindAnonimizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; export const assistantAnonimizationFieldsTypeName = 'elastic-ai-assistant-anonimization-fields'; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts index 5d909bd291215..2e1c67a02b936 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts @@ -12,8 +12,8 @@ import type { SavedObjectsType, SavedObjectsUpdateResponse, } from '@kbn/core/server'; -import { PromptResponse } from '../schemas/prompts/crud_prompts_route.gen'; -import { FindPromptsResponse } from '../schemas/prompts/find_prompts_route.gen'; +import { PromptResponse } from '@kbn/elastic-assistant-common'; +import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; export const assistantPromptsTypeName = 'elastic-ai-assistant-prompts'; diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index d13effa9a1f75..adb0ada80df26 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -26,12 +26,11 @@ import { AuthenticatedUser, SecurityPluginStart } from '@kbn/security-plugin/ser import { Tool } from 'langchain/dist/tools/base'; import { RetrievalQAChain } from 'langchain/chains'; import { ElasticsearchClient } from '@kbn/core/server'; -import { AssistantFeatures } from '@kbn/elastic-assistant-common'; +import { AssistantFeatures, ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; import { AIAssistantConversationsDataClient } from './conversations_data_client'; import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; import { AIAssistantAnonimizationFieldsSOClient } from './saved_object/ai_assistant_anonimization_fields_so_client'; -import { ExecuteConnectorRequestBody } from './schemas/actions_connector/post_actions_connector_execute_route.gen'; export const PLUGIN_ID = 'elasticAssistant' as const; diff --git a/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts index 7139192869106..f76adfdbd0894 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/alert_counts/alert_counts_tool.test.ts @@ -12,7 +12,7 @@ import { omit } from 'lodash/fp'; import { ALERT_COUNTS_TOOL } from './alert_counts_tool'; import type { RetrievalQAChain } from 'langchain/chains'; -import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('AlertCountsTool', () => { const alertsIndexPattern = 'alerts-index'; diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts index 82dff5bd8b52a..10da74669ea63 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts @@ -10,7 +10,7 @@ import type { DynamicTool } from 'langchain/tools'; import { ESQL_KNOWLEDGE_BASE_TOOL } from './esql_language_knowledge_base_tool'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { KibanaRequest } from '@kbn/core-http-server'; -import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('EsqlLanguageKnowledgeBaseTool', () => { const chain = {} as RetrievalQAChain; diff --git a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts index b90bd2fb0c096..d0dd3ed291648 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool.test.ts @@ -14,7 +14,7 @@ import { OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL } from './open_and_acknowledged_alert import { MAX_SIZE } from './helpers'; import type { RetrievalQAChain } from 'langchain/chains'; import { mockAlertsFieldsApi } from '@kbn/elastic-assistant-plugin/server/__mocks__/alerts'; -import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-plugin/server/schemas/actions_connector/post_actions_connector_execute_route.gen'; +import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen'; describe('OpenAndAcknowledgedAlertsTool', () => { const alertsIndexPattern = 'alerts-index'; From 1fbe6e223250d4f4ed3ae8b45662ccc299c469b6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 2 Feb 2024 02:00:19 +0000 Subject: [PATCH 063/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/packages/kbn-elastic-assistant-common/tsconfig.json | 3 +++ x-pack/plugins/elastic_assistant/tsconfig.json | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/tsconfig.json b/x-pack/packages/kbn-elastic-assistant-common/tsconfig.json index 94b099694eaf4..d4d082d9c13e5 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/tsconfig.json +++ b/x-pack/packages/kbn-elastic-assistant-common/tsconfig.json @@ -16,5 +16,8 @@ "target/**/*" ], "kbn_references": [ + "@kbn/zod-helpers", + "@kbn/securitysolution-io-ts-utils", + "@kbn/core", ] } diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 5ba0fea4969cb..48b3ac39e239a 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -47,7 +47,6 @@ "@kbn/core-saved-objects-server", "@kbn/spaces-plugin", "@kbn/zod-helpers", - "@kbn/securitysolution-io-ts-utils", ], "exclude": [ "target/**/*", From 5d49bedd9bcc450f867bc431e2844d64efbcbaae Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Thu, 1 Feb 2024 18:05:34 -0800 Subject: [PATCH 064/141] renamed anonimization to anonymizaton --- .../current_fields.json | 2 +- .../current_mappings.json | 2 +- .../check_registered_types.test.ts | 2 +- .../group3/type_registrations.test.ts | 2 +- .../group5/dot_kibana_split.test.ts | 2 +- .../kbn-elastic-assistant-common/constants.ts | 6 +- ...ulk_crud_anonymization_fields_route.gen.ts | 40 ++++---- ...rud_anonymization_fields_route.schema.yaml | 40 ++++---- .../find_anonymization_fields_route.gen.ts | 32 +++--- ...ind_anonymization_fields_route.schema.yaml | 20 ++-- .../elastic_assistant/common/constants.ts | 4 +- .../server/__mocks__/request_context.ts | 4 +- .../elastic_assistant/server/plugin.ts | 4 +- .../bulk_actions_route.ts | 34 +++---- .../routes/anonimization_fields/find_route.ts | 18 ++-- .../server/routes/request_context_factory.ts | 6 +- ...sistant_anonymization_fields_so_client.ts} | 98 +++++++++---------- ...ic_assistant_anonymization_fields_type.ts} | 36 +++---- .../server/saved_object/index.ts | 2 +- .../plugins/elastic_assistant/server/types.ts | 4 +- 20 files changed, 179 insertions(+), 179 deletions(-) rename x-pack/plugins/elastic_assistant/server/saved_object/{ai_assistant_anonimization_fields_so_client.ts => ai_assistant_anonymization_fields_so_client.ts} (70%) rename x-pack/plugins/elastic_assistant/server/saved_object/{elastic_assistant_anonimization_fields_type.ts => elastic_assistant_anonymization_fields_type.ts} (74%) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index cc485f7e27a6a..726d60857f9f1 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -987,7 +987,7 @@ "updated_at", "updated_by" ], - "elastic-ai-assistant-anonimization-fields": [ + "elastic-ai-assistant-anonymization-fields": [ "created_at", "created_by", "default_allow", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 112dcfc21700c..a20df64f3ff67 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2811,7 +2811,7 @@ } } }, - "elastic-ai-assistant-anonimization-fields": { + "elastic-ai-assistant-anonymization-fields": { "properties": { "id": { "type": "keyword" diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 74a713f7ec527..61e015c00ee49 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -83,7 +83,7 @@ describe('checking migration metadata changes on all registered SO types', () => "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "c151324d5f85178169395eecb12bac6b96064654", "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", - "elastic-ai-assistant-anonimization-fields": "04707fc69680fc95656f2438cdda1d70cbedf6bf", + "elastic-ai-assistant-anonymization-fields": "04707fc69680fc95656f2438cdda1d70cbedf6bf", "elastic-ai-assistant-prompts": "713a9d7e8f26b32ebb5c4042193ae29ba4059dd7", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index 527b0eff9ca16..ba87dc7086c74 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -44,7 +44,7 @@ const previouslyRegisteredTypes = [ 'csp-rule-template', 'csp_rule', 'dashboard', - 'elastic-ai-assistant-anonimization-fields', + 'elastic-ai-assistant-anonymization-fields', 'elastic-ai-assistant-prompts', 'event-annotation-group', 'endpoint:user-artifact', diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 5ecb474821ede..643dedb2bda0a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -203,7 +203,7 @@ describe('split .kibana index into multiple system indices', () => { "connector_token", "core-usage-stats", "csp-rule-template", - "elastic-ai-assistant-anonimization-fields", + "elastic-ai-assistant-anonymization-fields", "elastic-ai-assistant-prompts", "endpoint:user-artifact-manifest", "enterprise_search_telemetry", diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index ae6f951af72f0..31d803f66a5f4 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -23,6 +23,6 @@ export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/pro export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{id}`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_find`; -export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL = `${ELASTIC_AI_ASSISTANT_URL}/anonimization_fields`; -export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL}/_bulk_action`; -export const ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND = `${ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL}/_find`; +export const ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL = `${ELASTIC_AI_ASSISTANT_URL}/anonymization_fields`; +export const ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL}/_bulk_action`; +export const ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_FIND = `${ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL}/_find`; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts index 54418c17908e0..5cb2b3ce071e3 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts @@ -17,7 +17,7 @@ import { z } from 'zod'; */ export type BulkActionSkipReason = z.infer; -export const BulkActionSkipReason = z.literal('ANONIMIZATION_FIELD_NOT_MODIFIED'); +export const BulkActionSkipReason = z.literal('ANONYMIZATION_FIELD_NOT_MODIFIED'); export type BulkActionSkipResult = z.infer; export const BulkActionSkipResult = z.object({ @@ -26,22 +26,22 @@ export const BulkActionSkipResult = z.object({ skip_reason: BulkActionSkipReason, }); -export type AnonimizationFieldDetailsInError = z.infer; -export const AnonimizationFieldDetailsInError = z.object({ +export type AnonymizationFieldDetailsInError = z.infer; +export const AnonymizationFieldDetailsInError = z.object({ id: z.string(), name: z.string().optional(), }); -export type NormalizedAnonimizationFieldError = z.infer; -export const NormalizedAnonimizationFieldError = z.object({ +export type NormalizedAnonymizationFieldError = z.infer; +export const NormalizedAnonymizationFieldError = z.object({ message: z.string(), status_code: z.number().int(), err_code: z.string().optional(), - anonimization_fields: z.array(AnonimizationFieldDetailsInError), + anonymization_fields: z.array(AnonymizationFieldDetailsInError), }); -export type AnonimizationFieldResponse = z.infer; -export const AnonimizationFieldResponse = z.object({ +export type AnonymizationFieldResponse = z.infer; +export const AnonymizationFieldResponse = z.object({ id: z.string(), fieldId: z.string(), defaultAllow: z.boolean().optional(), @@ -54,8 +54,8 @@ export const AnonimizationFieldResponse = z.object({ export type BulkCrudActionResults = z.infer; export const BulkCrudActionResults = z.object({ - updated: z.array(AnonimizationFieldResponse), - created: z.array(AnonimizationFieldResponse), + updated: z.array(AnonymizationFieldResponse), + created: z.array(AnonymizationFieldResponse), deleted: z.array(z.string()), skipped: z.array(BulkActionSkipResult), }); @@ -73,35 +73,35 @@ export const BulkCrudActionResponse = z.object({ success: z.boolean().optional(), status_code: z.number().int().optional(), message: z.string().optional(), - anonimization_fields_count: z.number().int().optional(), + anonymization_fields_count: z.number().int().optional(), attributes: z.object({ results: BulkCrudActionResults, summary: BulkCrudActionSummary, - errors: z.array(NormalizedAnonimizationFieldError).optional(), + errors: z.array(NormalizedAnonymizationFieldError).optional(), }), }); export type BulkActionBase = z.infer; export const BulkActionBase = z.object({ /** - * Query to filter anonimization fields + * Query to filter anonymization fields */ query: z.string().optional(), /** - * Array of anonimization fields IDs + * Array of anonymization fields IDs */ ids: z.array(z.string()).min(1).optional(), }); -export type AnonimizationFieldCreateProps = z.infer; -export const AnonimizationFieldCreateProps = z.object({ +export type AnonymizationFieldCreateProps = z.infer; +export const AnonymizationFieldCreateProps = z.object({ fieldId: z.string(), defaultAllow: z.boolean().optional(), defaultAllowReplacement: z.boolean().optional(), }); -export type AnonimizationFieldUpdateProps = z.infer; -export const AnonimizationFieldUpdateProps = z.object({ +export type AnonymizationFieldUpdateProps = z.infer; +export const AnonymizationFieldUpdateProps = z.object({ id: z.string(), defaultAllow: z.boolean().optional(), defaultAllowReplacement: z.boolean().optional(), @@ -110,8 +110,8 @@ export const AnonimizationFieldUpdateProps = z.object({ export type PerformBulkActionRequestBody = z.infer; export const PerformBulkActionRequestBody = z.object({ delete: BulkActionBase.optional(), - create: z.array(AnonimizationFieldCreateProps).optional(), - update: z.array(AnonimizationFieldUpdateProps).optional(), + create: z.array(AnonymizationFieldCreateProps).optional(), + update: z.array(AnonymizationFieldUpdateProps).optional(), }); export type PerformBulkActionRequestBodyInput = z.input; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml index 204b0d8d93f20..0cf4a2746d33b 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml @@ -3,12 +3,12 @@ info: title: Bulk Actions API endpoint version: '2023-10-31' paths: - /api/elastic_assistant/anonimization_fields/_bulk_action: + /api/elastic_assistant/anonymization_fields/_bulk_action: post: operationId: PerformBulkAction x-codegen-enabled: true - summary: Applies a bulk action to multiple anonimization fields - description: The bulk action is applied to all anonimization fields that match the filter or to the list of anonimization fields by their IDs. + summary: Applies a bulk action to multiple anonymization fields + description: The bulk action is applied to all anonymization fields that match the filter or to the list of anonymization fields by their IDs. tags: - Bulk API requestBody: @@ -22,11 +22,11 @@ paths: create: type: array items: - $ref: '#/components/schemas/AnonimizationFieldCreateProps' + $ref: '#/components/schemas/AnonymizationFieldCreateProps' update: type: array items: - $ref: '#/components/schemas/AnonimizationFieldUpdateProps' + $ref: '#/components/schemas/AnonymizationFieldUpdateProps' responses: 200: description: Indicates a successful call. @@ -53,7 +53,7 @@ components: BulkActionSkipReason: type: string enum: - - ANONIMIZATION_FIELD_NOT_MODIFIED + - ANONYMIZATION_FIELD_NOT_MODIFIED BulkActionSkipResult: type: object @@ -68,7 +68,7 @@ components: - id - skip_reason - AnonimizationFieldDetailsInError: + AnonymizationFieldDetailsInError: type: object properties: id: @@ -78,7 +78,7 @@ components: required: - id - NormalizedAnonimizationFieldError: + NormalizedAnonymizationFieldError: type: object properties: message: @@ -87,16 +87,16 @@ components: type: integer err_code: type: string - anonimization_fields: + anonymization_fields: type: array items: - $ref: '#/components/schemas/AnonimizationFieldDetailsInError' + $ref: '#/components/schemas/AnonymizationFieldDetailsInError' required: - message - status_code - - anonimization_fields + - anonymization_fields - AnonimizationFieldResponse: + AnonymizationFieldResponse: type: object required: - id @@ -125,11 +125,11 @@ components: updated: type: array items: - $ref: '#/components/schemas/AnonimizationFieldResponse' + $ref: '#/components/schemas/AnonymizationFieldResponse' created: type: array items: - $ref: '#/components/schemas/AnonimizationFieldResponse' + $ref: '#/components/schemas/AnonymizationFieldResponse' deleted: type: array items: @@ -170,7 +170,7 @@ components: type: integer message: type: string - anonimization_fields_count: + anonymization_fields_count: type: integer attributes: type: object @@ -182,7 +182,7 @@ components: errors: type: array items: - $ref: '#/components/schemas/NormalizedAnonimizationFieldError' + $ref: '#/components/schemas/NormalizedAnonymizationFieldError' required: - results - summary @@ -196,15 +196,15 @@ components: properties: query: type: string - description: Query to filter anonimization fields + description: Query to filter anonymization fields ids: type: array - description: Array of anonimization fields IDs + description: Array of anonymization fields IDs minItems: 1 items: type: string - AnonimizationFieldCreateProps: + AnonymizationFieldCreateProps: type: object required: - fieldId @@ -216,7 +216,7 @@ components: defaultAllowReplacement: type: boolean - AnonimizationFieldUpdateProps: + AnonymizationFieldUpdateProps: type: object required: - id diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts index 04f48d117d658..58a1395c54a03 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen.ts @@ -13,31 +13,31 @@ import { ArrayFromString } from '@kbn/zod-helpers'; * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: Find AnonimizationFields API endpoint + * title: Find AnonymizationFields API endpoint * version: 2023-10-31 */ -import { AnonimizationFieldResponse } from './bulk_crud_anonymization_fields_route.gen'; +import { AnonymizationFieldResponse } from './bulk_crud_anonymization_fields_route.gen'; -export type FindAnonimizationFieldsSortField = z.infer; -export const FindAnonimizationFieldsSortField = z.enum([ +export type FindAnonymizationFieldsSortField = z.infer; +export const FindAnonymizationFieldsSortField = z.enum([ 'created_at', 'is_default', 'title', 'updated_at', ]); -export type FindAnonimizationFieldsSortFieldEnum = typeof FindAnonimizationFieldsSortField.enum; -export const FindAnonimizationFieldsSortFieldEnum = FindAnonimizationFieldsSortField.enum; +export type FindAnonymizationFieldsSortFieldEnum = typeof FindAnonymizationFieldsSortField.enum; +export const FindAnonymizationFieldsSortFieldEnum = FindAnonymizationFieldsSortField.enum; export type SortOrder = z.infer; export const SortOrder = z.enum(['asc', 'desc']); export type SortOrderEnum = typeof SortOrder.enum; export const SortOrderEnum = SortOrder.enum; -export type FindAnonimizationFieldsRequestQuery = z.infer< - typeof FindAnonimizationFieldsRequestQuery +export type FindAnonymizationFieldsRequestQuery = z.infer< + typeof FindAnonymizationFieldsRequestQuery >; -export const FindAnonimizationFieldsRequestQuery = z.object({ +export const FindAnonymizationFieldsRequestQuery = z.object({ fields: ArrayFromString(z.string()).optional(), /** * Search query @@ -46,7 +46,7 @@ export const FindAnonimizationFieldsRequestQuery = z.object({ /** * Field to sort by */ - sort_field: FindAnonimizationFieldsSortField.optional(), + sort_field: FindAnonymizationFieldsSortField.optional(), /** * Sort order */ @@ -56,18 +56,18 @@ export const FindAnonimizationFieldsRequestQuery = z.object({ */ page: z.coerce.number().int().min(1).optional().default(1), /** - * AnonimizationFields per page + * AnonymizationFields per page */ per_page: z.coerce.number().int().min(0).optional().default(20), }); -export type FindAnonimizationFieldsRequestQueryInput = z.input< - typeof FindAnonimizationFieldsRequestQuery +export type FindAnonymizationFieldsRequestQueryInput = z.input< + typeof FindAnonymizationFieldsRequestQuery >; -export type FindAnonimizationFieldsResponse = z.infer; -export const FindAnonimizationFieldsResponse = z.object({ +export type FindAnonymizationFieldsResponse = z.infer; +export const FindAnonymizationFieldsResponse = z.object({ page: z.number().int(), perPage: z.number().int(), total: z.number().int(), - data: z.array(AnonimizationFieldResponse), + data: z.array(AnonymizationFieldResponse), }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml index 5c0e7c951363d..3782fb2e4876a 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.schema.yaml @@ -1,16 +1,16 @@ openapi: 3.0.0 info: - title: Find AnonimizationFields API endpoint + title: Find AnonymizationFields API endpoint version: '2023-10-31' paths: - /api/elastic_assistant/anonimization_fields/_find: + /api/elastic_assistant/anonymization_fields/_find: get: - operationId: FindAnonimizationFields + operationId: FindAnonymizationFields x-codegen-enabled: true - description: Finds anonimization fields that match the given query. - summary: Finds anonimization fields that match the given query. + description: Finds anonymization fields that match the given query. + summary: Finds anonymization fields that match the given query. tags: - - AnonimizationFields API + - AnonymizationFields API parameters: - name: 'fields' in: query @@ -30,7 +30,7 @@ paths: description: Field to sort by required: false schema: - $ref: '#/components/schemas/FindAnonimizationFieldsSortField' + $ref: '#/components/schemas/FindAnonymizationFieldsSortField' - name: 'sort_order' in: query description: Sort order @@ -47,7 +47,7 @@ paths: default: 1 - name: 'per_page' in: query - description: AnonimizationFields per page + description: AnonymizationFields per page required: false schema: type: integer @@ -71,7 +71,7 @@ paths: data: type: array items: - $ref: './bulk_crud_anonimization_fields_route.schema.yaml#/components/schemas/AnonimizationFieldResponse' + $ref: './bulk_crud_anonymization_fields_route.schema.yaml#/components/schemas/AnonymizationFieldResponse' required: - page - perPage @@ -93,7 +93,7 @@ paths: components: schemas: - FindAnonimizationFieldsSortField: + FindAnonymizationFieldsSortField: type: string enum: - 'created_at' diff --git a/x-pack/plugins/elastic_assistant/common/constants.ts b/x-pack/plugins/elastic_assistant/common/constants.ts index fb3268211a922..8ed064c1744d0 100755 --- a/x-pack/plugins/elastic_assistant/common/constants.ts +++ b/x-pack/plugins/elastic_assistant/common/constants.ts @@ -21,8 +21,8 @@ export const EVALUATE = `${BASE_PATH}/evaluate`; export const MAX_CONVERSATIONS_TO_UPDATE_IN_PARALLEL = 50; export const CONVERSATIONS_TABLE_MAX_PAGE_SIZE = 100; -export const MAX_ANONIMIZATION_FIELDS_TO_UPDATE_IN_PARALLEL = 50; -export const ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE = 100; +export const MAX_ANONYMIZATION_FIELDS_TO_UPDATE_IN_PARALLEL = 50; +export const ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE = 100; // Capabilities export const CAPABILITIES = `${BASE_PATH}/capabilities`; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index f508fa047bd1e..9b34184d0c1f6 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -32,7 +32,7 @@ export const createMockClients = () => { telemetry: coreMock.createSetup().analytics, getAIAssistantConversationsDataClient: conversationsDataClientMock.create(), getAIAssistantPromptsSOClient: jest.fn(), - getAIAssistantAnonimizationFieldsSOClient: jest.fn(), + getAIAssistantAnonymizationFieldsSOClient: jest.fn(), getSpaceId: jest.fn(), getCurrentUser: jest.fn(), }, @@ -96,7 +96,7 @@ const createElasticAssistantRequestContextMock = ( (() => Promise), getAIAssistantPromptsSOClient: jest.fn(), - getAIAssistantAnonimizationFieldsSOClient: jest.fn(), + getAIAssistantAnonymizationFieldsSOClient: jest.fn(), getCurrentUser: jest.fn(), getServerBasePath: jest.fn(), getSpaceId: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index 0d3acd5f8f643..75ade90d7ea67 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -20,7 +20,7 @@ import { ElasticAssistantRequestHandlerContext, } from './types'; import { AIAssistantService } from './ai_assistant_service'; -import { assistantPromptsType, assistantAnonimizationFieldsType } from './saved_object'; +import { assistantPromptsType, assistantAnonymizationFieldsType } from './saved_object'; import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; import { registerRoutes } from './routes/register_routes'; @@ -78,7 +78,7 @@ export class ElasticAssistantPlugin events.forEach((eventConfig) => core.analytics.registerEventType(eventConfig)); core.savedObjects.registerType(assistantPromptsType); - core.savedObjects.registerType(assistantAnonimizationFieldsType); + core.savedObjects.registerType(assistantAnonymizationFieldsType); // this.assistantService registerKBTask registerRoutes(router, this.logger, plugins); diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts index 062dd7bd3a199..83520b06fcd05 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts @@ -10,13 +10,13 @@ import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/s import { transformError } from '@kbn/securitysolution-es-utils'; import { - ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; import { SavedObjectError } from '@kbn/core/types'; import { - AnonimizationFieldResponse, + AnonymizationFieldResponse, BulkActionSkipResult, BulkCrudActionResponse, BulkCrudActionResults, @@ -24,7 +24,7 @@ import { PerformBulkActionRequestBody, PerformBulkActionResponse, } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; -import { ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; +import { ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; @@ -32,7 +32,7 @@ import { buildResponse } from '../utils'; export interface BulkOperationError { message: string; status?: number; - anonimizationField: { + anonymizationField: { id: string; name: string; }; @@ -50,8 +50,8 @@ const buildBulkResponse = ( skipped = [], }: { errors?: SavedObjectError[]; - updated?: AnonimizationFieldResponse[]; - created?: AnonimizationFieldResponse[]; + updated?: AnonymizationFieldResponse[]; + created?: AnonymizationFieldResponse[]; deleted?: string[]; skipped?: BulkActionSkipResult[]; } @@ -92,21 +92,21 @@ const buildBulkResponse = ( const responseBody: BulkCrudActionResponse = { success: true, - anonimization_fields_count: summary.total, + anonymization_fields_count: summary.total, attributes: { results, summary }, }; return response.ok({ body: responseBody }); }; -export const bulkActionAnonimizationFieldsRoute = ( +export const bulkActionAnonymizationFieldsRoute = ( router: ElasticAssistantPluginRouter, logger: Logger ) => { router.versioned .post({ access: 'public', - path: ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_BULK_ACTION, + path: ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_BULK_ACTION, options: { tags: ['access:elasticAssistant'], timeout: { @@ -127,9 +127,9 @@ export const bulkActionAnonimizationFieldsRoute = ( const { body } = request; const assistantResponse = buildResponse(response); - if (body?.update && body.update?.length > ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE) { + if (body?.update && body.update?.length > ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE) { return assistantResponse.error({ - body: `More than ${ANONIMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, + body: `More than ${ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, statusCode: 400, }); } @@ -141,27 +141,27 @@ export const bulkActionAnonimizationFieldsRoute = ( request.events.completed$.subscribe(() => abortController.abort()); try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantAnonimizationFieldsSOClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsSOClient(); const docsCreated = body.create && body.create.length > 0 - ? await dataClient.createAnonimizationFields(body.create) + ? await dataClient.createAnonymizationFields(body.create) : []; const docsUpdated = body.update && body.update.length > 0 - ? await dataClient.updateAnonimizationFields(body.update) + ? await dataClient.updateAnonymizationFields(body.update) : []; - const docsDeleted = await dataClient.deleteAnonimizationFieldsByIds( + const docsDeleted = await dataClient.deleteAnonymizationFieldsByIds( body.delete?.ids ?? [] ); - const created = await dataClient?.findAnonimizationFields({ + const created = await dataClient?.findAnonymizationFields({ page: 1, perPage: 1000, filter: docsCreated.map((updatedId) => `id:${updatedId}`).join(' OR '), fields: ['id'], }); - const updated = await dataClient?.findAnonimizationFields({ + const updated = await dataClient?.findAnonymizationFields({ page: 1, perPage: 1000, filter: docsUpdated.map((updatedId) => `id:${updatedId}`).join(' OR '), diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts index 2ad72df91b362..a2de850544be8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts @@ -9,26 +9,26 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { - ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND, + ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_FIND, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; import { - FindAnonimizationFieldsRequestQuery, - FindAnonimizationFieldsResponse, + FindAnonymizationFieldsRequestQuery, + FindAnonymizationFieldsResponse, } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; -export const findAnonimizationFieldsRoute = ( +export const findAnonymizationFieldsRoute = ( router: ElasticAssistantPluginRouter, logger: Logger ) => { router.versioned .get({ access: 'public', - path: ELASTIC_AI_ASSISTANT_ANONIMIZATION_FIELDS_URL_FIND, + path: ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL_FIND, options: { tags: ['access:elasticAssistant'], }, @@ -38,7 +38,7 @@ export const findAnonimizationFieldsRoute = ( version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, validate: { request: { - query: buildRouteValidationWithZod(FindAnonimizationFieldsRequestQuery), + query: buildRouteValidationWithZod(FindAnonymizationFieldsRequestQuery), }, }, }, @@ -46,15 +46,15 @@ export const findAnonimizationFieldsRoute = ( context, request, response - ): Promise> => { + ): Promise> => { const assistantResponse = buildResponse(response); try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantAnonimizationFieldsSOClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsSOClient(); - const result = await dataClient?.findAnonimizationFields({ + const result = await dataClient?.findAnonymizationFields({ perPage: query.per_page, page: query.page, sortField: query.sort_field, diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 92ff2cf4e1fa9..440340db9747f 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -19,7 +19,7 @@ import { import { AIAssistantPromptsSOClient } from '../saved_object/ai_assistant_prompts_so_client'; import { AIAssistantService } from '../ai_assistant_service'; import { appContextService } from '../services/app_context'; -import { AIAssistantAnonimizationFieldsSOClient } from '../saved_object/ai_assistant_anonimization_fields_so_client'; +import { AIAssistantAnonymizationFieldsSOClient } from '../saved_object/ai_assistant_anonymization_fields_so_client'; export interface IRequestContextFactory { create( @@ -93,10 +93,10 @@ export class RequestContextFactory implements IRequestContextFactory { }); }), - getAIAssistantAnonimizationFieldsSOClient: memoize(() => { + getAIAssistantAnonymizationFieldsSOClient: memoize(() => { const username = startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; - return new AIAssistantAnonimizationFieldsSOClient({ + return new AIAssistantAnonymizationFieldsSOClient({ logger: options.logger, user: username, savedObjectsClient: coreContext.savedObjects.client, diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts similarity index 70% rename from x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts rename to x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts index fa588f118539d..b88fe52f2b893 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonimization_fields_so_client.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts @@ -13,45 +13,45 @@ import { } from '@kbn/core/server'; import { - AnonimizationFieldCreateProps, - AnonimizationFieldResponse, - AnonimizationFieldUpdateProps, + AnonymizationFieldCreateProps, + AnonymizationFieldResponse, + AnonymizationFieldUpdateProps, } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; import { - FindAnonimizationFieldsResponse, + FindAnonymizationFieldsResponse, SortOrder, } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; import { - AssistantAnonimizationFieldSoSchema, - assistantAnonimizationFieldsTypeName, - transformSavedObjectToAssistantAnonimizationField, - transformSavedObjectUpdateToAssistantAnonimizationField, - transformSavedObjectsToFoundAssistantAnonimizationField, -} from './elastic_assistant_anonimization_fields_type'; + AssistantAnonymizationFieldSoSchema, + assistantAnonymizationFieldsTypeName, + transformSavedObjectToAssistantAnonymizationField, + transformSavedObjectUpdateToAssistantAnonymizationField, + transformSavedObjectsToFoundAssistantAnonymizationField, +} from './elastic_assistant_anonymization_fields_type'; export interface ConstructorOptions { - /** User creating, modifying, deleting, or updating the anonimization fields */ + /** User creating, modifying, deleting, or updating the anonymization fields */ user: string; - /** Saved objects client to create, modify, delete, the anonimization fields */ + /** Saved objects client to create, modify, delete, the anonymization fields */ savedObjectsClient: SavedObjectsClientContract; logger: Logger; } /** - * Class for use for anonimization fields that are used for AI assistant. + * Class for use for anonymization fields that are used for AI assistant. */ -export class AIAssistantAnonimizationFieldsSOClient { - /** User creating, modifying, deleting, or updating the anonimization fields */ +export class AIAssistantAnonymizationFieldsSOClient { + /** User creating, modifying, deleting, or updating the anonymization fields */ private readonly user: string; - /** Saved objects client to create, modify, delete, the anonimization fields */ + /** Saved objects client to create, modify, delete, the anonymization fields */ private readonly savedObjectsClient: SavedObjectsClientContract; /** * Constructs the assistant client * @param options - * @param options.user The user associated with the anonimization fields - * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI anonimization fields + * @param options.user The user associated with the anonymization fields + * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI anonymization fields */ constructor({ user, savedObjectsClient }: ConstructorOptions) { this.user = user; @@ -59,20 +59,20 @@ export class AIAssistantAnonimizationFieldsSOClient { } /** - * Fetch an anonimization field + * Fetch an anonymization field * @param options * @param options.id the "id" of an exception list * @returns The found exception list or null if none exists */ - public getAnonimizationField = async (id: string): Promise => { + public getAnonymizationField = async (id: string): Promise => { const { savedObjectsClient } = this; if (id != null) { try { - const savedObject = await savedObjectsClient.get( - assistantAnonimizationFieldsTypeName, + const savedObject = await savedObjectsClient.get( + assistantAnonymizationFieldsTypeName, id ); - return transformSavedObjectToAssistantAnonimizationField({ savedObject }); + return transformSavedObjectToAssistantAnonymizationField({ savedObject }); } catch (err) { if (SavedObjectsErrorHelpers.isNotFoundError(err)) { return null; @@ -89,11 +89,11 @@ export class AIAssistantAnonimizationFieldsSOClient { * This creates an agnostic space endpoint list if it does not exist. This tries to be * as fast as possible by ignoring conflict errors and not returning the contents of the * list if it already exists. - * @returns AssistantAnonimizationFieldSchema if it created the endpoint list, otherwise null if it already exists + * @returns AssistantAnonymizationFieldSchema if it created the endpoint list, otherwise null if it already exists */ - public createAnonimizationFields = async ( - items: AnonimizationFieldCreateProps[] - ): Promise => { + public createAnonymizationFields = async ( + items: AnonymizationFieldCreateProps[] + ): Promise => { const { savedObjectsClient, user } = this; const dateNow = new Date().toISOString(); @@ -109,14 +109,14 @@ export class AIAssistantAnonimizationFieldsSOClient { updated_by: user, updated_at: dateNow, }, - type: assistantAnonimizationFieldsTypeName, + type: assistantAnonymizationFieldsTypeName, }; }); const savedObjectsBulk = - await savedObjectsClient.bulkCreate(formattedItems); + await savedObjectsClient.bulkCreate(formattedItems); const result = savedObjectsBulk.saved_objects.map((savedObject) => - transformSavedObjectToAssistantAnonimizationField({ savedObject }) + transformSavedObjectToAssistantAnonymizationField({ savedObject }) ); return result; } catch (err) { @@ -147,14 +147,14 @@ export class AIAssistantAnonimizationFieldsSOClient { * @param options.type The type of the endpoint list item (Default is "simple") * @returns The exception list item updated, otherwise null if not updated */ - public updateAnonimizationFields = async ( - items: AnonimizationFieldUpdateProps[] - ): Promise => { + public updateAnonymizationFields = async ( + items: AnonymizationFieldUpdateProps[] + ): Promise => { const { savedObjectsClient, user } = this; const dateNow = new Date().toISOString(); const existingItems = ( - await this.findAnonimizationFields({ + await this.findAnonymizationFields({ page: 1, perPage: 1000, filter: items.map((updated) => `id:${updated.id}`).join(' OR '), @@ -163,7 +163,7 @@ export class AIAssistantAnonimizationFieldsSOClient { ).data.reduce((res, item) => { res[item.id] = item; return res; - }, {} as Record); + }, {} as Record); const formattedItems = items.map((item) => { return { attributes: { @@ -173,35 +173,35 @@ export class AIAssistantAnonimizationFieldsSOClient { updated_at: dateNow, }, id: existingItems[item.id].id, - type: assistantAnonimizationFieldsTypeName, + type: assistantAnonymizationFieldsTypeName, }; }); const savedObjectsBulk = - await savedObjectsClient.bulkUpdate(formattedItems); + await savedObjectsClient.bulkUpdate(formattedItems); const result = savedObjectsBulk.saved_objects.map((savedObject) => - transformSavedObjectUpdateToAssistantAnonimizationField({ savedObject }) + transformSavedObjectUpdateToAssistantAnonymizationField({ savedObject }) ); return result; }; /** - * Delete the anonimization field by id + * Delete the anonymization field by id * @param options - * @param options.id the "id" of the anonimization field + * @param options.id the "id" of the anonymization field */ - public deleteAnonimizationFieldsByIds = async ( + public deleteAnonymizationFieldsByIds = async ( ids: string[] ): Promise => { const { savedObjectsClient } = this; const res = await savedObjectsClient.bulkDelete( - ids.map((id) => ({ id, type: assistantAnonimizationFieldsTypeName })) + ids.map((id) => ({ id, type: assistantAnonymizationFieldsTypeName })) ); return res.statuses; }; /** - * Finds anonimization fields given a set of criteria. + * Finds anonymization fields given a set of criteria. * @param options * @param options.filter The filter to apply in the search * @param options.perPage How many per page to return @@ -209,9 +209,9 @@ export class AIAssistantAnonimizationFieldsSOClient { * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in - * @returns The found anonimization fields or null if nothing is found + * @returns The found anonymization fields or null if nothing is found */ - public findAnonimizationFields = async ({ + public findAnonymizationFields = async ({ perPage, page, sortField, @@ -225,20 +225,20 @@ export class AIAssistantAnonimizationFieldsSOClient { sortOrder?: SortOrder; filter?: string; fields?: string[]; - }): Promise => { + }): Promise => { const { savedObjectsClient } = this; const savedObjectsFindResponse = - await savedObjectsClient.find({ + await savedObjectsClient.find({ filter, page, perPage, sortField, sortOrder, - type: assistantAnonimizationFieldsTypeName, + type: assistantAnonymizationFieldsTypeName, fields, }); - return transformSavedObjectsToFoundAssistantAnonimizationField({ savedObjectsFindResponse }); + return transformSavedObjectsToFoundAssistantAnonymizationField({ savedObjectsFindResponse }); }; } diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts similarity index 74% rename from x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts rename to x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts index b3aa65355bba0..a314e9ab83231 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonimization_fields_type.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts @@ -12,12 +12,12 @@ import type { SavedObjectsType, SavedObjectsUpdateResponse, } from '@kbn/core/server'; -import { AnonimizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; -import { FindAnonimizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; +import { AnonymizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; +import { FindAnonymizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; -export const assistantAnonimizationFieldsTypeName = 'elastic-ai-assistant-anonimization-fields'; +export const assistantAnonymizationFieldsTypeName = 'elastic-ai-assistant-anonymization-fields'; -export const assistantAnonimizationFieldsTypeMappings: SavedObjectsType['mappings'] = { +export const assistantAnonymizationFieldsTypeMappings: SavedObjectsType['mappings'] = { properties: { id: { type: 'keyword', @@ -46,11 +46,11 @@ export const assistantAnonimizationFieldsTypeMappings: SavedObjectsType['mapping }, }; -export const transformSavedObjectToAssistantAnonimizationField = ({ +export const transformSavedObjectToAssistantAnonymizationField = ({ savedObject, }: { - savedObject: SavedObject; -}): AnonimizationFieldResponse => { + savedObject: SavedObject; +}): AnonymizationFieldResponse => { const { version: _version, attributes: { @@ -79,7 +79,7 @@ export const transformSavedObjectToAssistantAnonimizationField = ({ }; }; -export interface AssistantAnonimizationFieldSoSchema { +export interface AssistantAnonymizationFieldSoSchema { created_at: string; created_by: string; field_id: string; @@ -89,19 +89,19 @@ export interface AssistantAnonimizationFieldSoSchema { updated_by: string; } -export const assistantAnonimizationFieldsType: SavedObjectsType = { - name: assistantAnonimizationFieldsTypeName, +export const assistantAnonymizationFieldsType: SavedObjectsType = { + name: assistantAnonymizationFieldsTypeName, indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, // todo: generic hidden: false, namespaceType: 'multiple-isolated', - mappings: assistantAnonimizationFieldsTypeMappings, + mappings: assistantAnonymizationFieldsTypeMappings, }; -export const transformSavedObjectUpdateToAssistantAnonimizationField = ({ +export const transformSavedObjectUpdateToAssistantAnonymizationField = ({ savedObject, }: { - savedObject: SavedObjectsUpdateResponse; -}): AnonimizationFieldResponse => { + savedObject: SavedObjectsUpdateResponse; +}): AnonymizationFieldResponse => { const dateNow = new Date().toISOString(); const { version: _version, @@ -129,14 +129,14 @@ export const transformSavedObjectUpdateToAssistantAnonimizationField = ({ }; }; -export const transformSavedObjectsToFoundAssistantAnonimizationField = ({ +export const transformSavedObjectsToFoundAssistantAnonymizationField = ({ savedObjectsFindResponse, }: { - savedObjectsFindResponse: SavedObjectsFindResponse; -}): FindAnonimizationFieldsResponse => { + savedObjectsFindResponse: SavedObjectsFindResponse; +}): FindAnonymizationFieldsResponse => { return { data: savedObjectsFindResponse.saved_objects.map((savedObject) => - transformSavedObjectToAssistantAnonimizationField({ savedObject }) + transformSavedObjectToAssistantAnonymizationField({ savedObject }) ), page: savedObjectsFindResponse.page, perPage: savedObjectsFindResponse.per_page, diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/index.ts b/x-pack/plugins/elastic_assistant/server/saved_object/index.ts index 3362a36721860..9d2ae737a4b07 100644 --- a/x-pack/plugins/elastic_assistant/server/saved_object/index.ts +++ b/x-pack/plugins/elastic_assistant/server/saved_object/index.ts @@ -6,4 +6,4 @@ */ export * from './elastic_assistant_prompts_type'; -export * from './elastic_assistant_anonimization_fields_type'; +export * from './elastic_assistant_anonymization_fields_type'; diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index adb0ada80df26..fe97c87539316 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -30,7 +30,7 @@ import { AssistantFeatures, ExecuteConnectorRequestBody } from '@kbn/elastic-ass import { AIAssistantConversationsDataClient } from './conversations_data_client'; import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; -import { AIAssistantAnonimizationFieldsSOClient } from './saved_object/ai_assistant_anonimization_fields_so_client'; +import { AIAssistantAnonymizationFieldsSOClient } from './saved_object/ai_assistant_anonymization_fields_so_client'; export const PLUGIN_ID = 'elasticAssistant' as const; @@ -100,7 +100,7 @@ export interface ElasticAssistantApiRequestHandlerContext { getCurrentUser: () => AuthenticatedUser | null; getAIAssistantConversationsDataClient: () => Promise; getAIAssistantPromptsSOClient: () => AIAssistantPromptsSOClient; - getAIAssistantAnonimizationFieldsSOClient: () => AIAssistantAnonimizationFieldsSOClient; + getAIAssistantAnonymizationFieldsSOClient: () => AIAssistantAnonymizationFieldsSOClient; telemetry: AnalyticsServiceSetup; } /** From 12c986e843a9090f90db25d95dc2928bb4ffe1e8 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 2 Feb 2024 02:31:37 +0000 Subject: [PATCH 065/141] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 61e015c00ee49..14b95909c16b1 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -83,7 +83,7 @@ describe('checking migration metadata changes on all registered SO types', () => "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "c151324d5f85178169395eecb12bac6b96064654", "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", - "elastic-ai-assistant-anonymization-fields": "04707fc69680fc95656f2438cdda1d70cbedf6bf", + "elastic-ai-assistant-anonymization-fields": "bf60d632de45cbeb69ab6b5f5579db608e67b97c", "elastic-ai-assistant-prompts": "713a9d7e8f26b32ebb5c4042193ae29ba4059dd7", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", From 5bfe84c1e10fce3bcaff37047266d334467a5618 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Mon, 5 Feb 2024 21:33:48 -0800 Subject: [PATCH 066/141] changed approach to use title for dictionary on the client side --- .../use_bulk_actions_conversations.ts | 52 +++++++--- .../impl/assistant/assistant_header/index.tsx | 9 +- .../assistant/assistant_overlay/index.tsx | 24 ++--- .../conversation_selector/index.test.tsx | 18 ++-- .../conversation_selector/index.tsx | 81 ++++++++------- .../index.test.tsx | 10 +- .../conversation_selector_settings/index.tsx | 62 ++++++------ .../conversation_settings.tsx | 15 +-- .../impl/assistant/helpers.ts | 14 +-- .../impl/assistant/index.test.tsx | 22 ++--- .../impl/assistant/index.tsx | 99 ++++++++++--------- .../system_prompt_settings.tsx | 4 +- .../assistant/settings/assistant_settings.tsx | 20 ++-- .../settings/assistant_settings_button.tsx | 2 +- .../use_settings_updater.tsx | 4 +- .../use_assistant_overlay/index.test.tsx | 2 +- .../assistant/use_assistant_overlay/index.tsx | 8 +- .../impl/assistant/use_conversation/index.tsx | 1 + .../impl/assistant_context/constants.tsx | 2 +- .../impl/assistant_context/index.test.tsx | 12 +-- .../impl/assistant_context/index.tsx | 29 +++--- .../connectorland/connector_setup/index.tsx | 2 +- .../impl/new_chat_by_id/index.test.tsx | 38 +++---- .../impl/new_chat_by_id/index.tsx | 20 ++-- .../impl/new_chat_by_id/translations.ts | 2 +- .../packages/kbn-elastic-assistant/index.ts | 8 +- .../append_conversation_messages.ts | 4 +- .../public/assistant/provider.tsx | 2 +- .../right/components/header_actions.tsx | 8 +- .../right/components/test_ids.ts | 2 +- .../event_details/expandable_event.tsx | 6 +- .../timeline/assistant_tab_content/index.tsx | 8 +- .../translations/translations/fr-FR.json | 2 +- .../translations/translations/ja-JP.json | 2 +- .../translations/translations/zh-CN.json | 2 +- 35 files changed, 325 insertions(+), 271 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 16d23964b507b..6592220c10f2d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -6,7 +6,8 @@ */ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; -import { HttpSetup } from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; +import { HttpSetup, IToasts } from '@kbn/core/public'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, @@ -44,7 +45,7 @@ export interface BulkActionResponse { success?: boolean; conversations_count?: number; message?: string; - status_code?: number; + statusCode?: number; attributes: BulkActionAttributes; } @@ -97,9 +98,10 @@ const transformUpdateActions = ( [] ); -export const bulkChangeConversations = ( +export const bulkChangeConversations = async ( http: HttpSetup, - conversationsActions: ConversationsBulkActions + conversationsActions: ConversationsBulkActions, + toasts?: IToasts ) => { // transform conversations disctionary to array of Conversations to create // filter marked as deleted @@ -113,13 +115,37 @@ export const bulkChangeConversations = ( ? transformUpdateActions(conversationsActions.update, conversationsActions.delete?.ids) : undefined; - return http.fetch(ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, { - method: 'POST', - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - body: JSON.stringify({ - update: conversationsToUpdate, - create: conversationsToCreate, - delete: conversationsActions.delete, - }), - }); + try { + const result = await http.fetch( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + { + method: 'POST', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + body: JSON.stringify({ + update: conversationsToUpdate, + create: conversationsToCreate, + delete: conversationsActions.delete, + }), + } + ); + if (!result.success) { + const serverError = result.attributes.errors + ?.map( + (e) => + `Error code: ${e.status_code}. Error message: ${ + e.message + } for conversation ${e.conversations.map((c) => c.name).join(',')}` + ) + .join(',\n'); + throw new Error(serverError); + } + return result; + } catch (error) { + toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { + title: i18n.translate('xpack.elasticAssistant.conversations.getConversationError', { + defaultMessage: 'Error updating conversations {error}', + values: { error }, + }), + }); + } }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index 35599e0ccc8c6..a1753662f333b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -31,7 +31,7 @@ interface OwnProps { docLinks: Omit; isDisabled: boolean; isSettingsModalVisible: boolean; - onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle: string }) => void; onConversationDeleted: (conversationId: string) => void; onToggleShowAnonymizedValues: (e: EuiSwitchEvent) => void; setIsSettingsModalVisible: React.Dispatch>; @@ -90,7 +90,10 @@ export const AssistantHeader: React.FC = ({ selectedConversation={currentConversation} onChange={(updatedConversation) => { setCurrentConversation(updatedConversation); - onConversationSelected({ cId: updatedConversation.id }); + onConversationSelected({ + cId: updatedConversation.id, + cTitle: updatedConversation.title, + }); }} title={title} /> @@ -105,7 +108,7 @@ export const AssistantHeader: React.FC = ({ { const [isModalVisible, setIsModalVisible] = useState(false); - const [conversationId, setConversationId] = useState( + const [conversationTitle, setConversationTitle] = useState( WELCOME_CONVERSATION_TITLE ); const [promptContextId, setPromptContextId] = useState(); - const { assistantTelemetry, setShowAssistantOverlay, getLastConversationId } = + const { assistantTelemetry, setShowAssistantOverlay, getLastConversationTitle } = useAssistantContext(); // Bind `showAssistantOverlay` in SecurityAssistantContext to this modal instance @@ -42,9 +42,9 @@ export const AssistantOverlay = React.memo(() => { ({ showOverlay: so, promptContextId: pid, - conversationId: cid, + conversationTitle: cTitle, }: ShowAssistantOverlayProps) => { - const newConversationId = getLastConversationId(cid); + const newConversationId = getLastConversationTitle(cTitle); if (so) assistantTelemetry?.reportAssistantInvoked({ conversationId: newConversationId, @@ -53,9 +53,9 @@ export const AssistantOverlay = React.memo(() => { setIsModalVisible(so); setPromptContextId(pid); - setConversationId(newConversationId); + setConversationTitle(newConversationId); }, - [assistantTelemetry, getLastConversationId] + [assistantTelemetry, getLastConversationTitle] ); useEffect(() => { setShowAssistantOverlay(showOverlay); @@ -65,15 +65,15 @@ export const AssistantOverlay = React.memo(() => { const handleShortcutPress = useCallback(() => { // Try to restore the last conversation on shortcut pressed if (!isModalVisible) { - setConversationId(getLastConversationId()); + setConversationTitle(getLastConversationTitle()); assistantTelemetry?.reportAssistantInvoked({ invokedBy: 'shortcut', - conversationId: getLastConversationId(), + conversationId: getLastConversationTitle(), }); } setIsModalVisible(!isModalVisible); - }, [isModalVisible, getLastConversationId, assistantTelemetry]); + }, [isModalVisible, getLastConversationTitle, assistantTelemetry]); // Register keyboard listener to show the modal when cmd + ; is pressed const onKeyDown = useCallback( @@ -91,8 +91,8 @@ export const AssistantOverlay = React.memo(() => { const cleanupAndCloseModal = useCallback(() => { setIsModalVisible(false); setPromptContextId(undefined); - setConversationId(conversationId); - }, [conversationId]); + setConversationTitle(conversationTitle); + }, [conversationTitle]); const handleCloseModal = useCallback(() => { cleanupAndCloseModal(); @@ -102,7 +102,7 @@ export const AssistantOverlay = React.memo(() => { <> {isModalVisible && ( - + )} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx index 27caa9d0dfb2b..9865e042f7b48 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.test.tsx @@ -26,14 +26,14 @@ const mockConversation = { }; const mockConversations = { - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, + [alertConvo.title]: alertConvo, + [welcomeConvo.title]: welcomeConvo, }; const mockConversationsWithCustom = { - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, + [alertConvo.title]: alertConvo, + [welcomeConvo.title]: welcomeConvo, + [customConvo.title]: customConvo, }; jest.mock('../../use_conversation', () => ({ @@ -45,7 +45,7 @@ const onConversationDeleted = jest.fn(); const defaultProps = { isDisabled: false, onConversationSelected, - selectedConversationId: 'Welcome', + selectedConversationTitle: 'Welcome', defaultConnectorId: '123', defaultProvider: OpenAiProviderType.OpenAi, conversations: mockConversations, @@ -146,7 +146,7 @@ describe('Conversation selector', () => { jest.runAllTimers(); expect(onConversationSelected).not.toHaveBeenCalled(); - expect(onConversationDeleted).toHaveBeenCalledWith(customConvo.id); + expect(onConversationDeleted).toHaveBeenCalledWith(customConvo.title); }); it('Previous conversation is set to active when selected conversation is deleted', () => { @@ -154,7 +154,7 @@ describe('Conversation selector', () => { ); @@ -231,7 +231,7 @@ describe('Conversation selector', () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index e7ee7656aec05..3b12c7245fd91 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -32,24 +32,30 @@ const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; interface Props { defaultConnectorId?: string; defaultProvider?: OpenAiProviderType; - selectedConversationId: string | undefined; - onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; + selectedConversationTitle: string | undefined; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle: string }) => void; onConversationDeleted: (conversationId: string) => void; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; conversations: Record; } -const getPreviousConversationId = (conversationIds: string[], selectedConversationId: string) => { - return conversationIds.indexOf(selectedConversationId) === 0 - ? conversationIds[conversationIds.length - 1] - : conversationIds[conversationIds.indexOf(selectedConversationId) - 1]; +const getPreviousConversationTitle = ( + conversationTitles: string[], + selectedConversationTitle: string +) => { + return conversationTitles.indexOf(selectedConversationTitle) === 0 + ? conversationTitles[conversationTitles.length - 1] + : conversationTitles[conversationTitles.indexOf(selectedConversationTitle) - 1]; }; -const getNextConversationId = (conversationIds: string[], selectedConversationId: string) => { - return conversationIds.indexOf(selectedConversationId) + 1 >= conversationIds.length - ? conversationIds[0] - : conversationIds[conversationIds.indexOf(selectedConversationId) + 1]; +const getNextConversationTitle = ( + conversationTitles: string[], + selectedConversationTitle: string +) => { + return conversationTitles.indexOf(selectedConversationTitle) + 1 >= conversationTitles.length + ? conversationTitles[0] + : conversationTitles[conversationTitles.indexOf(selectedConversationTitle) + 1]; }; export type ConversationSelectorOption = EuiComboBoxOptionOption<{ @@ -58,7 +64,7 @@ export type ConversationSelectorOption = EuiComboBoxOptionOption<{ export const ConversationSelector: React.FC = React.memo( ({ - selectedConversationId = DEFAULT_CONVERSATION_TITLE, + selectedConversationTitle = DEFAULT_CONVERSATION_TITLE, defaultConnectorId, defaultProvider, onConversationSelected, @@ -70,7 +76,7 @@ export const ConversationSelector: React.FC = React.memo( const { allSystemPrompts } = useAssistantContext(); const { createConversation } = useConversation(); - const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); + const conversationTitles = useMemo(() => Object.keys(conversations), [conversations]); const conversationOptions = useMemo(() => { return Object.values(conversations).map((conversation) => ({ value: { isDefault: conversation.isDefault ?? false }, @@ -80,7 +86,7 @@ export const ConversationSelector: React.FC = React.memo( }, [conversations]); const [selectedOptions, setSelectedOptions] = useState(() => { - return conversationOptions.filter((c) => c.id === selectedConversationId) ?? []; + return conversationOptions.filter((c) => c.label === selectedConversationTitle) ?? []; }); // Callback for when user types to create a new system prompt @@ -100,7 +106,7 @@ export const ConversationSelector: React.FC = React.memo( option.label.trim().toLowerCase() === normalizedSearchValue ) !== -1; - let cId; + let createdConversation; if (!optionExists) { const newConversation: Conversation = { id: searchValue, @@ -112,9 +118,13 @@ export const ConversationSelector: React.FC = React.memo( defaultSystemPromptId: defaultSystemPrompt?.id, }, }; - cId = (await createConversation(newConversation))?.id; + createdConversation = await createConversation(newConversation); } - onConversationSelected({ cId: cId ?? DEFAULT_CONVERSATION_TITLE }); + onConversationSelected( + createdConversation + ? { cId: createdConversation.id, cTitle: createdConversation.title } + : { cId: DEFAULT_CONVERSATION_TITLE, cTitle: DEFAULT_CONVERSATION_TITLE } + ); }, [ allSystemPrompts, @@ -129,19 +139,22 @@ export const ConversationSelector: React.FC = React.memo( const onDelete = useCallback( (cId: string) => { onConversationDeleted(cId); - if (selectedConversationId === cId) { - const prevConversationId = getPreviousConversationId(conversationIds, cId); + if (selectedConversationTitle === cId) { + const prevConversationTitle = getPreviousConversationTitle( + conversationTitles, + selectedConversationTitle + ); onConversationSelected({ - cId: prevConversationId, - cTitle: conversations[prevConversationId].title, + cId: conversations[prevConversationTitle].id, + cTitle: prevConversationTitle, }); } }, [ - selectedConversationId, + selectedConversationTitle, onConversationDeleted, onConversationSelected, - conversationIds, + conversationTitles, conversations, ] ); @@ -159,18 +172,18 @@ export const ConversationSelector: React.FC = React.memo( ); const onLeftArrowClick = useCallback(() => { - const prevId = getPreviousConversationId(conversationIds, selectedConversationId); - onConversationSelected({ cId: prevId, cTitle: conversations[prevId].title }); - }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); + const prevTitle = getPreviousConversationTitle(conversationTitles, selectedConversationTitle); + onConversationSelected({ cId: conversations[prevTitle].id, cTitle: prevTitle }); + }, [conversationTitles, selectedConversationTitle, onConversationSelected, conversations]); const onRightArrowClick = useCallback(() => { - const nextId = getNextConversationId(conversationIds, selectedConversationId); - onConversationSelected({ cId: nextId, cTitle: conversations[nextId].title }); - }, [conversationIds, selectedConversationId, onConversationSelected, conversations]); + const nextTitle = getNextConversationTitle(conversationTitles, selectedConversationTitle); + onConversationSelected({ cId: conversations[nextTitle].id, cTitle: nextTitle }); + }, [conversationTitles, selectedConversationTitle, onConversationSelected, conversations]); // Register keyboard listener for quick conversation switching const onKeyDown = useCallback( (event: KeyboardEvent) => { - if (isDisabled || conversationIds.length <= 1) { + if (isDisabled || conversationTitles.length <= 1) { return; } @@ -192,7 +205,7 @@ export const ConversationSelector: React.FC = React.memo( } }, [ - conversationIds.length, + conversationTitles.length, isDisabled, onLeftArrowClick, onRightArrowClick, @@ -202,8 +215,8 @@ export const ConversationSelector: React.FC = React.memo( useEvent('keydown', onKeyDown); useEffect(() => { - setSelectedOptions(conversationOptions.filter((c) => c.id === selectedConversationId)); - }, [conversationOptions, selectedConversationId]); + setSelectedOptions(conversationOptions.filter((c) => c.label === selectedConversationTitle)); + }, [conversationOptions, selectedConversationTitle]); const renderOption: ( option: ConversationSelectorOption, @@ -289,7 +302,7 @@ export const ConversationSelector: React.FC = React.memo( iconType="arrowLeft" aria-label={i18n.PREVIOUS_CONVERSATION_TITLE} onClick={onLeftArrowClick} - disabled={isDisabled || conversationIds.length <= 1} + disabled={isDisabled || conversationTitles.length <= 1} /> } @@ -299,7 +312,7 @@ export const ConversationSelector: React.FC = React.memo( iconType="arrowRight" aria-label={i18n.NEXT_CONVERSATION_TITLE} onClick={onRightArrowClick} - disabled={isDisabled || conversationIds.length <= 1} + disabled={isDisabled || conversationTitles.length <= 1} /> } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.test.tsx index 150784d9db4cf..b9df58fed2375 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.test.tsx @@ -13,13 +13,13 @@ import { alertConvo, customConvo, welcomeConvo } from '../../../mock/conversatio const onConversationSelectionChange = jest.fn(); const onConversationDeleted = jest.fn(); const mockConversations = { - [alertConvo.id]: alertConvo, - [welcomeConvo.id]: welcomeConvo, - [customConvo.id]: customConvo, + [alertConvo.title]: alertConvo, + [welcomeConvo.title]: welcomeConvo, + [customConvo.title]: customConvo, }; const testProps = { conversations: mockConversations, - selectedConversationId: welcomeConvo.id, + selectedConversationTitle: welcomeConvo.id, onConversationDeleted, onConversationSelectionChange, }; @@ -40,7 +40,7 @@ describe('ConversationSelectorSettings', () => { fireEvent.click(getByTestId('comboBoxToggleListButton')); // there is only one delete conversation because there is only one custom convo fireEvent.click(getByTestId('delete-conversation')); - expect(onConversationDeleted).toHaveBeenCalledWith(customConvo.id); + expect(onConversationDeleted).toHaveBeenCalledWith(customConvo.title); }); it('Selects existing conversation from the search input', () => { const { getByTestId } = render(); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index 3cc929de3f980..2306b74ebca4d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -24,23 +24,29 @@ import { SystemPromptSelectorOption } from '../../prompt_editor/system_prompt/sy interface Props { conversations: Record; - onConversationDeleted: (conversationId: string) => void; + onConversationDeleted: (conversationTitle: string) => void; onConversationSelectionChange: (conversation?: Conversation | string) => void; - selectedConversationId?: string; + selectedConversationTitle: string; shouldDisableKeyboardShortcut?: () => boolean; isDisabled?: boolean; } -const getPreviousConversationId = (conversationIds: string[], selectedConversationId = '') => { - return conversationIds.indexOf(selectedConversationId) === 0 - ? conversationIds[conversationIds.length - 1] - : conversationIds[conversationIds.indexOf(selectedConversationId) - 1]; +const getPreviousConversationTitle = ( + conversationTitles: string[], + selectedConversationTitle: string +) => { + return conversationTitles.indexOf(selectedConversationTitle) === 0 + ? conversationTitles[conversationTitles.length - 1] + : conversationTitles[conversationTitles.indexOf(selectedConversationTitle) - 1]; }; -const getNextConversationId = (conversationIds: string[], selectedConversationId = '') => { - return conversationIds.indexOf(selectedConversationId) + 1 >= conversationIds.length - ? conversationIds[0] - : conversationIds[conversationIds.indexOf(selectedConversationId) + 1]; +const getNextConversationTitle = ( + conversationTitles: string[], + selectedConversationTitle: string +) => { + return conversationTitles.indexOf(selectedConversationTitle) + 1 >= conversationTitles.length + ? conversationTitles[0] + : conversationTitles[conversationTitles.indexOf(selectedConversationTitle) + 1]; }; export type ConversationSelectorSettingsOption = EuiComboBoxOptionOption<{ @@ -58,11 +64,11 @@ export const ConversationSelectorSettings: React.FC = React.memo( conversations, onConversationDeleted, onConversationSelectionChange, - selectedConversationId, + selectedConversationTitle, isDisabled, shouldDisableKeyboardShortcut = () => false, }) => { - const conversationIds = useMemo(() => Object.keys(conversations), [conversations]); + const conversationTitles = useMemo(() => Object.keys(conversations), [conversations]); const [conversationOptions, setConversationOptions] = useState< ConversationSelectorSettingsOption[] @@ -76,10 +82,10 @@ export const ConversationSelectorSettings: React.FC = React.memo( }); const selectedOptions = useMemo(() => { - return selectedConversationId - ? conversationOptions.filter((c) => c.id === selectedConversationId) ?? [] + return selectedConversationTitle + ? conversationOptions.filter((c) => c.label === selectedConversationTitle) ?? [] : []; - }, [conversationOptions, selectedConversationId]); + }, [conversationOptions, selectedConversationTitle]); const handleSelectionChange = useCallback( (conversationSelectorSettingsOption: ConversationSelectorSettingsOption[]) => { @@ -137,26 +143,26 @@ export const ConversationSelectorSettings: React.FC = React.memo( // Callback for when user deletes a conversation const onDelete = useCallback( - (id: string) => { - setConversationOptions(conversationOptions.filter((o) => o.id !== id)); - if (selectedOptions?.[0]?.id === id) { + (title: string) => { + setConversationOptions(conversationOptions.filter((o) => o.label !== title)); + if (selectedOptions?.[0]?.title === title) { handleSelectionChange([]); } - onConversationDeleted(id); + onConversationDeleted(title); }, [conversationOptions, handleSelectionChange, onConversationDeleted, selectedOptions] ); const onLeftArrowClick = useCallback(() => { - const prevId = getPreviousConversationId(conversationIds, selectedConversationId); - const previousOption = conversationOptions.filter((c) => c.id === prevId); + const prevTitle = getPreviousConversationTitle(conversationTitles, selectedConversationTitle); + const previousOption = conversationOptions.filter((c) => c.label === prevTitle); handleSelectionChange(previousOption); - }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); + }, [conversationTitles, selectedConversationTitle, conversationOptions, handleSelectionChange]); const onRightArrowClick = useCallback(() => { - const nextId = getNextConversationId(conversationIds, selectedConversationId); - const nextOption = conversationOptions.filter((c) => c.id === nextId); + const nextTitle = getNextConversationTitle(conversationTitles, selectedConversationTitle); + const nextOption = conversationOptions.filter((c) => c.label === nextTitle); handleSelectionChange(nextOption); - }, [conversationIds, conversationOptions, handleSelectionChange, selectedConversationId]); + }, [conversationTitles, selectedConversationTitle, conversationOptions, handleSelectionChange]); const renderOption: ( option: ConversationSelectorSettingsOption, @@ -198,7 +204,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( data-test-subj="delete-conversation" onClick={(e: React.MouseEvent) => { e.stopPropagation(); - onDelete(id ?? ''); + onDelete(label); }} css={css` visibility: hidden; @@ -241,7 +247,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( data-test-subj="arrowLeft" aria-label={i18n.PREVIOUS_CONVERSATION_TITLE} onClick={onLeftArrowClick} - disabled={isDisabled || conversationIds.length <= 1} + disabled={isDisabled || conversationTitles.length <= 1} /> } append={ @@ -250,7 +256,7 @@ export const ConversationSelectorSettings: React.FC = React.memo( data-test-subj="arrowRight" aria-label={i18n.NEXT_CONVERSATION_TITLE} onClick={onRightArrowClick} - disabled={isDisabled || conversationIds.length <= 1} + disabled={isDisabled || conversationTitles.length <= 1} /> } /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index 8374dfe1d3f9a..1c30be18e477f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -92,7 +92,7 @@ export const ConversationSettings: React.FC = React.m ) { setConversationSettings({ ...conversationSettings, - [isNew ? c : newSelectedConversation.id]: newSelectedConversation, + [isNew ? c : newSelectedConversation.title]: newSelectedConversation, }); setConversationsSettingsBulkActions({ ...conversationsSettingsBulkActions, @@ -105,7 +105,7 @@ export const ConversationSettings: React.FC = React.m setConversationSettings((prev) => { return { ...prev, - [newSelectedConversation.id]: newSelectedConversation, + [newSelectedConversation.title]: newSelectedConversation, }; }); } @@ -125,9 +125,10 @@ export const ConversationSettings: React.FC = React.m ); const onConversationDeleted = useCallback( - (conversationId: string) => { + (conversationTitle: string) => { + const conversationId = conversationSettings[conversationTitle].id; const updatedConverationSettings = { ...conversationSettings }; - delete updatedConverationSettings[conversationId]; + delete updatedConverationSettings[conversationTitle]; setConversationSettings(updatedConverationSettings); setConversationsSettingsBulkActions({ @@ -157,7 +158,7 @@ export const ConversationSettings: React.FC = React.m }; setConversationSettings({ ...conversationSettings, - [updatedConversation.id]: updatedConversation, + [updatedConversation.title]: updatedConversation, }); if (selectedConversation.id !== selectedConversation.title) { setConversationsSettingsBulkActions({ @@ -226,7 +227,7 @@ export const ConversationSettings: React.FC = React.m }; setConversationSettings({ ...conversationSettings, - [selectedConversation.id]: updatedConversation, + [selectedConversation.title]: updatedConversation, }); if (selectedConversation.id !== selectedConversation.title) { setConversationsSettingsBulkActions({ @@ -338,7 +339,7 @@ export const ConversationSettings: React.FC = React.m => { const userConversations = (conversationsData?.data ?? []).reduce>( (transformed, conversation) => { - transformed[conversation.id] = conversation; + transformed[conversation.title] = conversation; return transformed; }, {} ); - return merge( - userConversations, - Object.keys(baseConversations) - .filter( - (baseId) => (conversationsData?.data ?? []).find((c) => c.title === baseId) === undefined - ) - .reduce>((transformed, conversation) => { - transformed[conversation] = baseConversations[conversation]; - return transformed; - }, {}) - ); + return merge(baseConversations, userConversations); }; export const getBlockBotConversation = ( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index 0700ccd40dbfd..d00f6926a58f5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -65,7 +65,7 @@ describe('Assistant', () => { }); describe('when selected conversation changes and some connectors are loaded', () => { - it('should persist the conversation id to local storage', async () => { + it('should persist the conversation title to local storage', async () => { const connectors: unknown[] = [{}]; jest.mocked(useLoadConnectors).mockReturnValue({ @@ -76,13 +76,13 @@ describe('Assistant', () => { jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ data: { Welcome: { - id: 'Welcome', + id: 'Welcome Id', title: 'Welcome', messages: [], apiConfig: {}, }, 'electric sheep': { - id: 'electric sheep', + id: 'electric sheep id', title: 'electric sheep', messages: [], apiConfig: {}, @@ -119,13 +119,13 @@ describe('Assistant', () => { jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ data: { Welcome: { - id: 'Welcome', + id: 'Welcome Id', title: 'Welcome', messages: [], apiConfig: {}, }, 'electric sheep': { - id: 'electric sheep', + id: 'electric sheep id', title: 'electric sheep', messages: [], apiConfig: {}, @@ -151,9 +151,9 @@ describe('Assistant', () => { }); expect(persistToLocalStorage).toHaveBeenLastCalledWith(WELCOME_CONVERSATION_TITLE); }); - it('should call the setConversationId callback if it is defined and the conversation id changes', async () => { + it('should call the setConversationTitle callback if it is defined and the conversation id changes', async () => { const connectors: unknown[] = [{}]; - const setConversationId = jest.fn(); + const setConversationTitle = jest.fn(); jest.mocked(useLoadConnectors).mockReturnValue({ isSuccess: true, data: connectors, @@ -162,13 +162,13 @@ describe('Assistant', () => { jest.mocked(useFetchCurrentUserConversations).mockReturnValue({ data: { Welcome: { - id: 'Welcome', + id: 'Welcome Id', title: 'Welcome', messages: [], apiConfig: {}, }, 'electric sheep': { - id: 'electric sheep', + id: 'electric sheep id', title: 'electric sheep', messages: [], apiConfig: {}, @@ -178,13 +178,13 @@ describe('Assistant', () => { refetch: jest.fn(), } as unknown as UseQueryResult, unknown>); - renderAssistant({ setConversationId }); + renderAssistant({ setConversationTitle }); await act(async () => { fireEvent.click(screen.getByLabelText('Previous conversation')); }); - expect(setConversationId).toHaveBeenLastCalledWith('electric sheep'); + expect(setConversationTitle).toHaveBeenLastCalledWith('electric sheep'); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index a994afe227816..94cfd0c9d6c5e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -61,12 +61,12 @@ import { Conversation } from '../assistant_context/types'; import { clearPresentationData } from '../connectorland/connector_setup/helpers'; export interface Props { - conversationId?: string; + conversationTitle?: string; embeddedLayout?: boolean; promptContextId?: string; shouldRefocusPrompt?: boolean; showTitle?: boolean; - setConversationId?: Dispatch>; + setConversationTitle?: Dispatch>; } /** @@ -74,12 +74,12 @@ export interface Props { * quick prompts for common actions, settings, and prompt context providers. */ const AssistantComponent: React.FC = ({ - conversationId, + conversationTitle, embeddedLayout = false, promptContextId = '', shouldRefocusPrompt = false, showTitle = true, - setConversationId, + setConversationTitle, }) => { const { assistantTelemetry, @@ -91,8 +91,8 @@ const AssistantComponent: React.FC = ({ getComments, http, promptContexts, - setLastConversationId, - getLastConversationId, + setLastConversationTitle, + getLastConversationTitle, title, allSystemPrompts, baseConversations, @@ -150,46 +150,49 @@ const AssistantComponent: React.FC = ({ [connectors] ); - const [selectedConversationId, setSelectedConversationId] = useState( - isAssistantEnabled ? getLastConversationId(conversationId) : WELCOME_CONVERSATION_TITLE + const [selectedConversationTitle, setSelectedConversationTitle] = useState( + isAssistantEnabled ? getLastConversationTitle(conversationTitle) : WELCOME_CONVERSATION_TITLE ); useEffect(() => { - if (setConversationId) { - setConversationId(selectedConversationId); + if (setConversationTitle) { + setConversationTitle(selectedConversationTitle); } - }, [selectedConversationId, setConversationId]); + }, [selectedConversationTitle, setConversationTitle]); const [currentConversation, setCurrentConversation] = useState( - getDefaultConversation({ conversationId: selectedConversationId }) + getDefaultConversation({ conversationId: selectedConversationTitle }) ); const refetchCurrentConversation = useCallback( async (cId?: string) => { - if ( - (!cId && selectedConversationId === currentConversation.title) || - !conversations[selectedConversationId] - ) { + if (!cId || cId === selectedConversationTitle || !conversations[selectedConversationTitle]) { return; } - const updatedConversation = await getConversation(cId ?? selectedConversationId); + const updatedConversation = await getConversation(cId ?? selectedConversationTitle); if (updatedConversation) { setCurrentConversation(updatedConversation); } return updatedConversation; }, - [conversations, currentConversation.title, getConversation, selectedConversationId] + [conversations, getConversation, selectedConversationTitle] ); useEffect(() => { if (!isLoading) { const conversation = - conversations[selectedConversationId ?? getLastConversationId(conversationId)]; + conversations[selectedConversationTitle ?? getLastConversationTitle(conversationTitle)]; if (conversation) { setCurrentConversation(conversation); } } - }, [conversationId, conversations, getLastConversationId, isLoading, selectedConversationId]); + }, [ + conversationTitle, + conversations, + getLastConversationTitle, + isLoading, + selectedConversationTitle, + ]); // Welcome setup state const isWelcomeSetup = useMemo(() => { @@ -216,11 +219,11 @@ const AssistantComponent: React.FC = ({ // Clear it if there is no connectors useEffect(() => { if (areConnectorsFetched && !connectors?.length) { - return setLastConversationId(WELCOME_CONVERSATION_TITLE); + return setLastConversationTitle(WELCOME_CONVERSATION_TITLE); } if (!currentConversation.excludeFromLastConversationStorage) { - setLastConversationId(currentConversation.id); + setLastConversationTitle(currentConversation.title); } }, [ areConnectorsFetched, @@ -228,7 +231,7 @@ const AssistantComponent: React.FC = ({ conversationsData, currentConversation, isLoading, - setLastConversationId, + setLastConversationTitle, ]); const [promptTextPreview, setPromptTextPreview] = useState(''); @@ -297,31 +300,31 @@ const AssistantComponent: React.FC = ({ ); const handleOnConversationSelected = useCallback( - async ({ cId, cTitle }: { cId: string; cTitle?: string }) => { - if (conversations[cId] === undefined && cId) { + async ({ cId, cTitle }: { cId: string; cTitle: string }) => { + if (cTitle === cId) { const updatedConv = await refetchResults(); if (updatedConv) { - setCurrentConversation(updatedConv[cId]); - setSelectedConversationId(cId); + setCurrentConversation(updatedConv[cTitle]); + setSelectedConversationTitle(cTitle); setEditingSystemPromptId( - getDefaultSystemPrompt({ allSystemPrompts, conversation: updatedConv[cId] })?.id + getDefaultSystemPrompt({ allSystemPrompts, conversation: updatedConv[cTitle] })?.id ); } - } else if (cId && cId !== cTitle) { - setSelectedConversationId(cId); + } else if (cId !== cTitle) { + setSelectedConversationTitle(cTitle); const refetchedConversation = await refetchCurrentConversation(cId); if (refetchedConversation) { setCurrentConversation(refetchedConversation); - setConversations({ ...(conversations ?? {}), [cId]: refetchedConversation }); + setConversations({ ...(conversations ?? {}), [cTitle]: refetchedConversation }); } setEditingSystemPromptId( getDefaultSystemPrompt({ allSystemPrompts, conversation: refetchedConversation })?.id ); } else { - setSelectedConversationId(cId); - setCurrentConversation(conversations[cId]); + setSelectedConversationTitle(cTitle); + setCurrentConversation(conversations[cTitle]); setEditingSystemPromptId( - getDefaultSystemPrompt({ allSystemPrompts, conversation: conversations[cId] })?.id + getDefaultSystemPrompt({ allSystemPrompts, conversation: conversations[cTitle] })?.id ); } }, @@ -340,12 +343,12 @@ const AssistantComponent: React.FC = ({ }); const handleOnConversationDeleted = useCallback( - async (cId: string) => { + async (cTitle: string) => { setTimeout(() => { - deleteConversation(cId); + deleteConversation(conversations[cTitle].id); }, 0); const deletedConv = { ...conversations }; - delete deletedConv[cId]; + delete deletedConv[cTitle]; setConversations(deletedConv); }, [conversations, deleteConversation] @@ -371,8 +374,8 @@ const AssistantComponent: React.FC = ({ ); useEffect(() => { - // Adding `conversationId !== selectedConversationId` to prevent auto-run still executing after changing selected conversation - if (currentConversation.messages.length || conversationId !== selectedConversationId) { + // Adding `conversationTitle !== selectedConversationTitle` to prevent auto-run still executing after changing selected conversation + if (currentConversation.messages.length || conversationTitle !== selectedConversationTitle) { return; } @@ -409,8 +412,8 @@ const AssistantComponent: React.FC = ({ currentConversation.messages, promptContexts, promptContextId, - conversationId, - selectedConversationId, + conversationTitle, + selectedConversationTitle, selectedPromptContexts, autoPopulatedOnce, defaultAllow, @@ -537,27 +540,27 @@ const AssistantComponent: React.FC = ({ const trackPrompt = useCallback( (promptTitle: string) => { assistantTelemetry?.reportAssistantQuickPrompt({ - conversationId: selectedConversationId, + conversationId: currentConversation.title, promptTitle, }); }, - [assistantTelemetry, selectedConversationId] + [assistantTelemetry, currentConversation.title] ); const refetchConversationsState = useCallback(async () => { const refetchedConversations = await refetchResults(); - if (refetchedConversations && refetchedConversations[selectedConversationId]) { - setCurrentConversation(refetchedConversations[selectedConversationId]); + if (refetchedConversations && refetchedConversations[currentConversation.title]) { + setCurrentConversation(refetchedConversations[currentConversation.title]); } else if (refetchedConversations) { const createdSelectedConversation = Object.values(refetchedConversations).find( - (c) => c.title === selectedConversationId + (c) => c.title === currentConversation.title ); if (createdSelectedConversation) { setCurrentConversation(createdSelectedConversation); - setSelectedConversationId(createdSelectedConversation.id); + setSelectedConversationTitle(createdSelectedConversation.title); } } - }, [refetchResults, selectedConversationId]); + }, [currentConversation.title, refetchResults]); return getWrapper( <> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx index de672294aac07..b847d62e802b1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx @@ -118,7 +118,7 @@ export const SystemPromptSettings: React.FC = React.memo( if (selectedSystemPrompt != null) { setConversationSettings((prev) => keyBy( - 'id', + 'title', /* * updatedConversationWithPrompts calculates the present of prompt for * each conversation. Based on the values of selected conversation, it goes @@ -160,7 +160,7 @@ export const SystemPromptSettings: React.FC = React.memo( : {}; const updateOperation = - convo.id === convo.title + convo.id !== convo.title ? { update: { ...(updatedConversationsSettingsBulkActions.update ?? {}), diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index 4e386557db633..fac45abc6e718 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -65,7 +65,7 @@ interface Props { ) => void; onSave: () => Promise; selectedConversation: Conversation; - onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle: string }) => void; conversations: Record; } @@ -108,7 +108,7 @@ export const AssistantSettings: React.FC = React.memo( // Conversation Selection State const [selectedConversation, setSelectedConversation] = useState( () => { - return conversationSettings[defaultSelectedConversation.id]; + return conversationSettings[defaultSelectedConversation.title]; } ); const onHandleSelectedConversationChange = useCallback((conversation?: Conversation) => { @@ -116,7 +116,7 @@ export const AssistantSettings: React.FC = React.memo( }, []); useEffect(() => { if (selectedConversation != null) { - setSelectedConversation(conversationSettings[selectedConversation.id]); + setSelectedConversation(conversationSettings[selectedConversation.title]); } }, [conversationSettings, selectedConversation]); @@ -147,16 +147,20 @@ export const AssistantSettings: React.FC = React.memo( const handleSave = useCallback(async () => { // If the selected conversation is deleted, we need to select a new conversation to prevent a crash creating a conversation that already exists const isSelectedConversationDeleted = - conversationSettings[defaultSelectedConversation.id] == null; - const newSelectedConversationId: string | undefined = Object.keys(conversationSettings)[0]; - if (isSelectedConversationDeleted && newSelectedConversationId != null) { - onConversationSelected({ cId: newSelectedConversationId }); + conversationSettings[defaultSelectedConversation.title] == null; + const newSelectedConversation: Conversation | undefined = + Object.values(conversationSettings)[0]; + if (isSelectedConversationDeleted && newSelectedConversation != null) { + onConversationSelected({ + cId: newSelectedConversation.id, + cTitle: newSelectedConversation.title, + }); } await saveSettings(); await onSave(); }, [ conversationSettings, - defaultSelectedConversation.id, + defaultSelectedConversation.title, onConversationSelected, onSave, saveSettings, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index acec91064d72a..6755017de4311 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -20,7 +20,7 @@ interface Props { isSettingsModalVisible: boolean; selectedConversation: Conversation; setIsSettingsModalVisible: React.Dispatch>; - onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle?: string }) => void; + onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle: string }) => void; isDisabled?: boolean; conversations: Record; refetchConversationsState: () => Promise; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 1b161d1497331..3b358093208c4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -52,6 +52,7 @@ export const useSettingsUpdater = ( setDefaultAllowReplacement, setKnowledgeBase, http, + toasts, } = useAssistantContext(); /** @@ -102,7 +103,7 @@ export const useSettingsUpdater = ( const saveSettings = useCallback(async (): Promise => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - await bulkChangeConversations(http, conversationsSettingsBulkActions); + await bulkChangeConversations(http, conversationsSettingsBulkActions, toasts); const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; @@ -128,6 +129,7 @@ export const useSettingsUpdater = ( updatedSystemPromptSettings, http, conversationsSettingsBulkActions, + toasts, knowledgeBase.isEnabledKnowledgeBase, knowledgeBase.isEnabledRAGAlerts, updatedKnowledgeBaseSettings, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.test.tsx index 4329cc45b565c..328577a37db2f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.test.tsx @@ -96,7 +96,7 @@ describe('useAssistantOverlay', () => { expect(mockUseAssistantContext.showAssistantOverlay).toHaveBeenCalledWith({ showOverlay: true, promptContextId: 'id', - conversationId: 'conversation-id', + conversationTitle: 'conversation-id', }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.tsx index ba1992b5e50de..16f372b415208 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_assistant_overlay/index.tsx @@ -20,7 +20,7 @@ interface UseAssistantOverlay { * `useAssistantOverlay` is a hook that registers context with the assistant overlay, and * returns an optional `showAssistantOverlay` function to display the assistant overlay. * As an alterative to using the `showAssistantOverlay` returned from this hook, you may - * use the `NewChatById` component and pass it the `promptContextId` returned by this hook. + * use the `NewChatByTitle` component and pass it the `promptContextId` returned by this hook. * * USE THIS WHEN: You want to register context in one part of the tree, and then show * a _New chat_ button in another part of the tree without passing around the data, or when @@ -38,7 +38,7 @@ export const useAssistantOverlay = ( /** * optionally automatically add this context to a specific conversation when the assistant is displayed */ - conversationId: string | null, + conversationTitle: string | null, /** * The assistant will display this **short**, static description @@ -98,11 +98,11 @@ export const useAssistantOverlay = ( assistantContextShowOverlay({ showOverlay, promptContextId, - conversationId: conversationId ?? undefined, + conversationTitle: conversationTitle ?? undefined, }); } }, - [assistantContextShowOverlay, conversationId, promptContextId] + [assistantContextShowOverlay, conversationTitle, promptContextId] ); useEffect(() => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index b258f706f8cd2..59f248a50aef5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -219,6 +219,7 @@ export const useConversation = (): UseConversation => { defaultSystemPromptId, }, id: conversationId, + title: conversationId, messages: messages != null ? messages : [], }; return newConversation; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx index cc747a705b851..b818cd40b85ec 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/constants.tsx @@ -10,7 +10,7 @@ import { KnowledgeBaseConfig } from '../assistant/types'; export const DEFAULT_ASSISTANT_NAMESPACE = 'elasticAssistantDefault'; export const QUICK_PROMPT_LOCAL_STORAGE_KEY = 'quickPrompts'; export const SYSTEM_PROMPT_LOCAL_STORAGE_KEY = 'systemPrompts'; -export const LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY = 'lastConversationId'; +export const LAST_CONVERSATION_TITLE_LOCAL_STORAGE_KEY = 'lastConversationTitle'; export const KNOWLEDGE_BASE_LOCAL_STORAGE_KEY = 'knowledgeBase'; /** The default `n` latest alerts, ordered by risk score, sent as context to the assistant */ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx index ccfc53cbaacb6..b2a41b9179c98 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.test.tsx @@ -35,22 +35,22 @@ describe('AssistantContext', () => { expect(result.current.http.fetch).toBeCalledWith(path); }); - test('getLastConversationId defaults to provided id', async () => { + test('getLastConversationTitle defaults to provided id', async () => { const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getLastConversationId('123'); + const id = result.current.getLastConversationTitle('123'); expect(id).toEqual('123'); }); - test('getLastConversationId uses local storage id when no id is provided ', async () => { + test('getLastConversationTitle uses local storage id when no id is provided ', async () => { const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getLastConversationId(); + const id = result.current.getLastConversationTitle(); expect(id).toEqual('456'); }); - test('getLastConversationId defaults to Welcome when no local storage id and no id is provided ', async () => { + test('getLastConversationTitle defaults to Welcome when no local storage id and no id is provided ', async () => { (useLocalStorage as jest.Mock).mockReturnValue([undefined, jest.fn()]); const { result } = renderHook(useAssistantContext, { wrapper: TestProviders }); - const id = result.current.getLastConversationId(); + const id = result.current.getLastConversationTitle(); expect(id).toEqual('Welcome'); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 4837d744447bc..5c888abe50537 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -31,7 +31,7 @@ import { DEFAULT_ASSISTANT_NAMESPACE, DEFAULT_KNOWLEDGE_BASE_SETTINGS, KNOWLEDGE_BASE_LOCAL_STORAGE_KEY, - LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY, + LAST_CONVERSATION_TITLE_LOCAL_STORAGE_KEY, QUICK_PROMPT_LOCAL_STORAGE_KEY, SYSTEM_PROMPT_LOCAL_STORAGE_KEY, } from './constants'; @@ -43,13 +43,13 @@ import { WELCOME_CONVERSATION_TITLE } from '../assistant/use_conversation/transl export interface ShowAssistantOverlayProps { showOverlay: boolean; promptContextId?: string; - conversationId?: string; + conversationTitle?: string; } type ShowAssistantOverlay = ({ showOverlay, promptContextId, - conversationId, + conversationTitle, }: ShowAssistantOverlayProps) => void; export interface AssistantProviderProps { actionTypeRegistry: ActionTypeRegistryContract; @@ -134,7 +134,7 @@ export interface UseAssistantContext { }) => EuiCommentProps[]; http: HttpSetup; knowledgeBase: KnowledgeBaseConfig; - getLastConversationId: (id?: string) => string; + getLastConversationTitle: (conversationTitle?: string) => string; promptContexts: Record; modelEvaluatorEnabled: boolean; nameSpace: string; @@ -145,7 +145,7 @@ export interface UseAssistantContext { setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; setKnowledgeBase: React.Dispatch>; - setLastConversationId: React.Dispatch>; + setLastConversationTitle: React.Dispatch>; setSelectedSettingsTab: React.Dispatch>; setShowAssistantOverlay: (showAssistantOverlay: ShowAssistantOverlay) => void; showAssistantOverlay: ShowAssistantOverlay; @@ -197,8 +197,8 @@ export const AssistantProvider: React.FC = ({ baseSystemPrompts ); - const [localStorageLastConversationId, setLocalStorageLastConversationId] = - useLocalStorage(`${nameSpace}.${LAST_CONVERSATION_ID_LOCAL_STORAGE_KEY}`); + const [localStorageLastConversationTitle, setLocalStorageLastConversationTitle] = + useLocalStorage(`${nameSpace}.${LAST_CONVERSATION_TITLE_LOCAL_STORAGE_KEY}`); /** * Local storage for knowledge base configuration, prefixed by assistant nameSpace @@ -253,12 +253,13 @@ export const AssistantProvider: React.FC = ({ */ const [selectedSettingsTab, setSelectedSettingsTab] = useState(CONVERSATIONS_TAB); - const getLastConversationId = useCallback( + const getLastConversationTitle = useCallback( // if a conversationId has been provided, use that // if not, check local storage // last resort, go to welcome conversation - (id?: string) => id ?? localStorageLastConversationId ?? WELCOME_CONVERSATION_TITLE, - [localStorageLastConversationId] + (conversationTitle?: string) => + conversationTitle ?? localStorageLastConversationTitle ?? WELCOME_CONVERSATION_TITLE, + [localStorageLastConversationTitle] ); // Fetch assistant capabilities @@ -304,8 +305,8 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, - getLastConversationId, - setLastConversationId: setLocalStorageLastConversationId, + getLastConversationTitle, + setLastConversationTitle: setLocalStorageLastConversationTitle, baseConversations, }), [ @@ -343,8 +344,8 @@ export const AssistantProvider: React.FC = ({ title, toasts, unRegisterPromptContext, - getLastConversationId, - setLocalStorageLastConversationId, + getLastConversationTitle, + setLocalStorageLastConversationTitle, baseConversations, ] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index de0ed6374fe98..cb451fb704f85 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -38,7 +38,7 @@ const SkipEuiText = styled(EuiText)` export interface ConnectorSetupProps { conversation?: Conversation; onSetupComplete?: () => void; - onConversationUpdate: ({ cId, cTitle }: { cId: string; cTitle?: string }) => Promise; + onConversationUpdate: ({ cId, cTitle }: { cId: string; cTitle: string }) => Promise; } export const useConnectorSetup = ({ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx index f8b1a81387b17..4ded731612a56 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx @@ -9,7 +9,7 @@ import React from 'react'; import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; -import { NewChatById } from '.'; +import { NewChatByTitle } from '.'; const mockUseAssistantContext = { showAssistantOverlay: jest.fn(), @@ -18,71 +18,73 @@ jest.mock('../assistant_context', () => ({ useAssistantContext: () => mockUseAssistantContext, })); -describe('NewChatById', () => { +describe('NewChatByTitle', () => { afterEach(() => { jest.clearAllMocks(); }); it('renders the default New Chat button with a discuss icon', () => { - render(); + render(); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.querySelector('[data-euiicon-type="discuss"]')).toBeInTheDocument(); }); it('renders the default "New Chat" text when children are NOT provided', () => { - render(); + render(); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.textContent).toContain('Chat'); }); it('renders custom children', async () => { - render({'🪄✨'}); + render({'🪄✨'}); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.textContent).toContain('🪄✨'); }); it('renders custom icons', async () => { - render(); + render(); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.querySelector('[data-euiicon-type="help"]')).toBeInTheDocument(); }); it('does NOT render an icon when iconType is null', () => { - render(); + render(); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.querySelector('.euiButtonContent__icon')).not.toBeInTheDocument(); }); it('renders button icon when iconOnly is true', async () => { - render(); + render(); - const newChatButton = screen.getByTestId('newChatById'); + const newChatButton = screen.getByTestId('newChatByTitle'); expect(newChatButton.querySelector('[data-euiicon-type="discuss"]')).toBeInTheDocument(); expect(newChatButton.textContent).not.toContain('Chat'); }); it('calls showAssistantOverlay on click', () => { - const conversationId = 'test-conversation-id'; + const conversationTitle = 'test-conversation-id'; const promptContextId = 'test-prompt-context-id'; - render(); - const newChatButton = screen.getByTestId('newChatById'); + render( + + ); + const newChatButton = screen.getByTestId('newChatByTitle'); userEvent.click(newChatButton); expect(mockUseAssistantContext.showAssistantOverlay).toHaveBeenCalledWith({ - conversationId, + conversationTitle, promptContextId, showOverlay: true, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx index 4e52018fa9c63..3ea4007eeef51 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx @@ -15,7 +15,7 @@ import * as i18n from './translations'; export interface Props { children?: React.ReactNode; /** Optionally automatically add this context to a conversation when the assistant is shown */ - conversationId?: string; + conversationTitle?: string; /** Defaults to `discuss`. If null, the button will not have an icon */ iconType?: string | null; /** Optionally specify a well known ID, or default to a UUID */ @@ -24,9 +24,9 @@ export interface Props { iconOnly?: boolean; } -const NewChatByIdComponent: React.FC = ({ +const NewChatByTitleComponent: React.FC = ({ children = i18n.NEW_CHAT, - conversationId, + conversationTitle, iconType, promptContextId, iconOnly = false, @@ -36,11 +36,11 @@ const NewChatByIdComponent: React.FC = ({ // proxy show / hide calls to assistant context, using our internal prompt context id: const showOverlay = useCallback(() => { showAssistantOverlay({ - conversationId, + conversationTitle, promptContextId, showOverlay: true, }); - }, [conversationId, promptContextId, showAssistantOverlay]); + }, [conversationTitle, promptContextId, showAssistantOverlay]); const icon = useMemo(() => { if (iconType === null) { @@ -55,7 +55,7 @@ const NewChatByIdComponent: React.FC = ({ iconOnly ? ( = ({ ) : ( = ({ ); }; -NewChatByIdComponent.displayName = 'NewChatByIdComponent'; +NewChatByTitleComponent.displayName = 'NewChatByTitleComponent'; /** - * `NewChatByID` displays a _New chat_ icon button by providing only the `promptContextId` + * `NewChatByTitle` displays a _New chat_ icon button by providing only the `promptContextId` * of a context that was (already) registered by the `useAssistantOverlay` hook. You may * optionally style the button icon, or override the default _New chat_ text with custom * content, like {'🪄✨'} @@ -90,4 +90,4 @@ NewChatByIdComponent.displayName = 'NewChatByIdComponent'; * registered where the data is available, and then the _New chat_ button can be displayed * in another part of the tree. */ -export const NewChatById = React.memo(NewChatByIdComponent); +export const NewChatByTitle = React.memo(NewChatByTitleComponent); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts index 1cf3ec2609a73..57de1f990dc6c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; export const NEW_CHAT = i18n.translate( - 'xpack.elasticAssistant.assistant.newChatById.newChatByIdButton', + 'xpack.elasticAssistant.assistant.newChatByTitle.newChatByTitleButton', { defaultMessage: 'Chat', } diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index d02836f04a38d..5c32597135569 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -28,7 +28,7 @@ export { Assistant } from './impl/assistant'; // Step 3: Wherever you want to bring context into the assistant, use the any combination of the following // components and hooks: // - `NewChat` component -// - `NewChatById` component +// - `NewChatByTitle` component // - `useAssistantOverlay` hook /** @@ -42,7 +42,7 @@ export { Assistant } from './impl/assistant'; export { NewChat } from './impl/new_chat'; /** - * `NewChatByID` displays a _New chat_ icon button by providing only the `promptContextId` + * `NewChatByTitle` displays a _New chat_ icon button by providing only the `promptContextId` * of a context that was (already) registered by the `useAssistantOverlay` hook. You may * optionally style the button icon, or override the default _New chat_ text with custom * content, like {'🪄✨'} @@ -53,13 +53,13 @@ export { NewChat } from './impl/new_chat'; * registered where the data is available, and then the _New chat_ button can be displayed * in another part of the tree. */ -export { NewChatById } from './impl/new_chat_by_id'; +export { NewChatByTitle } from './impl/new_chat_by_id'; /** * `useAssistantOverlay` is a hook that registers context with the assistant overlay, and * returns an optional `showAssistantOverlay` function to display the assistant overlay. * As an alterative to using the `showAssistantOverlay` returned from this hook, you may - * use the `NewChatById` component and pass it the `promptContextId` returned by this hook. + * use the `NewChatByTitle` component and pass it the `promptContextId` returned by this hook. * * USE THIS WHEN: You want to register context in one part of the tree, and then show * a _New chat_ button in another part of the tree without passing around the data, or when diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index 76a9360fbb1db..3e08c145f529b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -69,7 +69,7 @@ export const appendConversationMessages = async ({ }, }); if (response.failures && response.failures.length > 0) { - logger.warn( + logger.error( `Error appending conversation messages: ${response.failures.map( (f) => f.id )} for conversation by ID: ${existingConversation.id}` @@ -86,7 +86,7 @@ export const appendConversationMessages = async ({ }); return updatedConversation; } catch (err) { - logger.warn( + logger.error( `Error appending conversation messages: ${err} for conversation by ID: ${existingConversation.id}` ); throw err; diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 5bc05de5295e5..6fa412f40e28d 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -91,7 +91,7 @@ export const AssistantProvider: React.FC = ({ children }) => { return res; }, {}), }); - if (bulkResult.success) { + if (bulkResult && bulkResult.success) { storage.remove(`securitySolution.${LOCAL_STORAGE_KEY}`); } } diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/header_actions.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/header_actions.tsx index 52bcb514e7cab..2c48989f54de1 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/header_actions.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/header_actions.tsx @@ -9,7 +9,7 @@ import type { VFC } from 'react'; import React, { memo } from 'react'; import { EuiButtonIcon, EuiCopy, EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { NewChatById } from '@kbn/elastic-assistant'; +import { NewChatByTitle } from '@kbn/elastic-assistant'; import { URL_PARAM_KEY } from '../../../../common/hooks/use_url_state'; import { copyFunction } from '../../../shared/utils/copy_to_clipboard'; import { useGetAlertDetailsFlyoutLink } from '../../../../timelines/components/side_panel/event_details/use_get_alert_details_flyout_link'; @@ -57,8 +57,10 @@ export const HeaderActions: VFC = memo(() => { > {showAssistant && ( - diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/test_ids.ts b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/test_ids.ts index 2a218e9b28234..7ddef9ac16016 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/test_ids.ts +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/test_ids.ts @@ -19,7 +19,7 @@ export const SEVERITY_VALUE_TEST_ID = 'severity' as const; export const RISK_SCORE_TITLE_TEST_ID = `${FLYOUT_HEADER_TEST_ID}RiskScoreTitle` as const; export const RISK_SCORE_VALUE_TEST_ID = `${FLYOUT_HEADER_TEST_ID}RiskScoreValue` as const; export const SHARE_BUTTON_TEST_ID = `${FLYOUT_HEADER_TEST_ID}ShareButton` as const; -export const CHAT_BUTTON_TEST_ID = 'newChatById' as const; +export const CHAT_BUTTON_TEST_ID = 'newChatByTitle' as const; export const ASSIGNEES_HEADER_TEST_ID = `${FLYOUT_HEADER_TEST_ID}AssigneesHeader` as const; export const ASSIGNEES_TITLE_TEST_ID = `${FLYOUT_HEADER_TEST_ID}AssigneesTitle` as const; diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx index 2d4d493c5f8ca..d46c7e9f4206f 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { NewChatById } from '@kbn/elastic-assistant'; +import { NewChatByTitle } from '@kbn/elastic-assistant'; import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; import { isEmpty } from 'lodash/fp'; import { @@ -154,8 +154,8 @@ export const ExpandableEventTitle = React.memo( {hasAssistantPrivilege && promptContextId != null && ( - >; -}> = memo(({ shouldRefocusPrompt, setConversationId }) => ( + setConversationTitle: Dispatch>; +}> = memo(({ shouldRefocusPrompt, setConversationTitle }) => ( diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 03bc0ba726407..a8140ce46e297 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -12548,7 +12548,7 @@ "xpack.elasticAssistant.assistant.firstPromptEditor.selectASystemPromptPlaceholder": "Sélectionner une invite système", "xpack.elasticAssistant.assistant.firstPromptEditor.youLabel": "Vous", "xpack.elasticAssistant.assistant.newChat.newChatButton": "Chat", - "xpack.elasticAssistant.assistant.newChatById.newChatByIdButton": "Chat", + "xpack.elasticAssistant.assistant.newChatByTitle.newChatByTitleButton": "Chat", "xpack.elasticAssistant.assistant.overlay.CancelButton": "Annuler", "xpack.elasticAssistant.assistant.promptEditor.selectedPromotContexts.removeContextTooltip": "Retirer le contexte", "xpack.elasticAssistant.assistant.promptEditor.systemPrompt.settings.defaultConversationsHelpText": "Conversations devant utiliser cette invite système par défaut", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index fa24ab1e8141c..74d00cea4ccd2 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -12561,7 +12561,7 @@ "xpack.elasticAssistant.assistant.firstPromptEditor.selectASystemPromptPlaceholder": "システムプロンプトを選択", "xpack.elasticAssistant.assistant.firstPromptEditor.youLabel": "あなた", "xpack.elasticAssistant.assistant.newChat.newChatButton": "チャット", - "xpack.elasticAssistant.assistant.newChatById.newChatByIdButton": "チャット", + "xpack.elasticAssistant.assistant.newChatByTitle.newChatByTitleButton": "チャット", "xpack.elasticAssistant.assistant.overlay.CancelButton": "キャンセル", "xpack.elasticAssistant.assistant.promptEditor.selectedPromotContexts.removeContextTooltip": "コンテキストを削除", "xpack.elasticAssistant.assistant.promptEditor.systemPrompt.settings.defaultConversationsHelpText": "デフォルトでこのシステムプロンプトを使用する会話", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a59e1fc17ac2c..79565f3692745 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -12655,7 +12655,7 @@ "xpack.elasticAssistant.assistant.firstPromptEditor.selectASystemPromptPlaceholder": "选择系统提示", "xpack.elasticAssistant.assistant.firstPromptEditor.youLabel": "您", "xpack.elasticAssistant.assistant.newChat.newChatButton": "聊天", - "xpack.elasticAssistant.assistant.newChatById.newChatByIdButton": "聊天", + "xpack.elasticAssistant.assistant.newChatByTitle.newChatByTitleButton": "聊天", "xpack.elasticAssistant.assistant.overlay.CancelButton": "取消", "xpack.elasticAssistant.assistant.promptEditor.selectedPromotContexts.removeContextTooltip": "删除上下文", "xpack.elasticAssistant.assistant.promptEditor.systemPrompt.settings.defaultConversationsHelpText": "应默认使用此系统提示的对话", From c3534b426437ec1f18bd9e863f3647e6564de7a6 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 6 Feb 2024 12:28:01 -0800 Subject: [PATCH 067/141] Change routes to serve only current user --- .../kbn-elastic-assistant-common/constants.ts | 4 +- .../use_bulk_actions_conversations.ts | 2 +- .../use_fetch_current_user_conversations.ts | 6 +- .../assistant/settings/assistant_settings.tsx | 6 +- .../settings/assistant_settings_button.tsx | 21 +- .../use_settings_updater.tsx | 12 +- .../append_conversation_messages.ts | 5 +- .../conversations_data_writer.test.ts | 11 + .../conversations_data_writer.ts | 294 ++++++++++++------ .../create_conversation.ts | 13 +- .../delete_conversation.test.ts | 10 +- .../delete_conversation.ts | 37 +-- .../get_conversation.test.ts | 13 +- .../get_conversation.ts | 42 ++- .../conversations_data_client/index.test.ts | 10 +- .../server/conversations_data_client/index.ts | 57 ++-- .../update_conversation.ts | 4 +- .../routes/conversations/find_route.test.ts | 108 ------- .../server/routes/conversations/find_route.ts | 65 ---- .../server/routes/register_routes.ts | 30 +- ...append_conversation_messages_route.test.ts | 9 + .../append_conversation_messages_route.ts | 16 +- .../bulk_actions_route.test.ts | 21 +- .../bulk_actions_route.ts | 31 +- .../create_route.test.ts | 18 ++ .../create_route.ts | 15 +- .../delete_route.test.ts | 9 + .../delete_route.ts | 9 +- .../find_user_conversations_route.test.ts | 8 +- .../find_user_conversations_route.ts | 4 +- .../read_route.test.ts | 18 ++ .../read_route.ts | 9 +- .../update_route.test.ts | 9 + .../update_route.ts | 16 +- .../public/assistant/provider.tsx | 30 +- 35 files changed, 569 insertions(+), 403 deletions(-) delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/append_conversation_messages_route.test.ts (93%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/append_conversation_messages_route.ts (87%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/bulk_actions_route.test.ts (90%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/bulk_actions_route.ts (83%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/create_route.test.ts (88%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/create_route.ts (84%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/delete_route.test.ts (91%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/delete_route.ts (87%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/find_user_conversations_route.test.ts (90%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/find_user_conversations_route.ts (94%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/read_route.test.ts (82%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/read_route.ts (88%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/update_route.test.ts (94%) rename x-pack/plugins/elastic_assistant/server/routes/{conversations => user_conversations}/update_route.ts (87%) diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index 31d803f66a5f4..183197850e53b 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -10,14 +10,12 @@ export const ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION = '1'; export const ELASTIC_AI_ASSISTANT_URL = '/api/elastic_assistant'; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/conversations`; -export const ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/current_user`; +export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL = `${ELASTIC_AI_ASSISTANT_URL}/current_user/conversations`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/{id}`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID}/messages`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_bulk_action`; export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; -export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS = `${ELASTIC_AI_ASSISTANT_CURRENT_USER_CONVERSATIONS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/prompts`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{id}`; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 6592220c10f2d..01cb406935d37 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -142,7 +142,7 @@ export const bulkChangeConversations = async ( return result; } catch (error) { toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { - title: i18n.translate('xpack.elasticAssistant.conversations.getConversationError', { + title: i18n.translate('xpack.elasticAssistant.conversations.bulkActionsConversationsError', { defaultMessage: 'Error updating conversations {error}', values: { error }, }), diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts index 7ad0257d25e4e..5a1478e6725b3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.ts @@ -8,7 +8,7 @@ import { HttpSetup } from '@kbn/core/public'; import { useQuery } from '@tanstack/react-query'; import { - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; import { Conversation } from '../../../assistant_context/types'; @@ -47,7 +47,7 @@ export const useFetchCurrentUserConversations = ({ }; const cachingKeys = [ - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, query.page, query.perPage, ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, @@ -55,7 +55,7 @@ export const useFetchCurrentUserConversations = ({ return useQuery([cachingKeys, query], async () => { const res = await http.fetch( - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, { method: 'GET', version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index fac45abc6e718..fbaae94ef0138 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -63,7 +63,7 @@ interface Props { onClose: ( event?: React.KeyboardEvent | React.MouseEvent ) => void; - onSave: () => Promise; + onSave: (success: boolean) => Promise; selectedConversation: Conversation; onConversationSelected: ({ cId, cTitle }: { cId: string; cTitle: string }) => void; conversations: Record; @@ -156,8 +156,8 @@ export const AssistantSettings: React.FC = React.memo( cTitle: newSelectedConversation.title, }); } - await saveSettings(); - await onSave(); + const saveResult = await saveSettings(); + await onSave(saveResult); }, [ conversationSettings, defaultSelectedConversation.title, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx index 6755017de4311..b89f837431e0f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx @@ -52,14 +52,19 @@ export const AssistantSettingsButton: React.FC = React.memo( cleanupAndCloseModal(); }, [cleanupAndCloseModal]); - const handleSave = useCallback(async () => { - cleanupAndCloseModal(); - await refetchConversationsState(); - toasts?.addSuccess({ - iconType: 'check', - title: i18n.SETTINGS_UPDATED_TOAST_TITLE, - }); - }, [cleanupAndCloseModal, refetchConversationsState, toasts]); + const handleSave = useCallback( + async (success: boolean) => { + cleanupAndCloseModal(); + await refetchConversationsState(); + if (success) { + toasts?.addSuccess({ + iconType: 'check', + title: i18n.SETTINGS_UPDATED_TOAST_TITLE, + }); + } + }, + [cleanupAndCloseModal, refetchConversationsState, toasts] + ); const handleShowConversationSettings = useCallback(() => { setSelectedSettingsTab(CONVERSATIONS_TAB); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx index 3b358093208c4..5886fe967837f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.tsx @@ -32,7 +32,7 @@ interface UseSettingsUpdater { setUpdatedKnowledgeBaseSettings: React.Dispatch>; setUpdatedQuickPromptSettings: React.Dispatch>; setUpdatedSystemPromptSettings: React.Dispatch>; - saveSettings: () => Promise; + saveSettings: () => Promise; } export const useSettingsUpdater = ( @@ -100,10 +100,14 @@ export const useSettingsUpdater = ( /** * Save all pending settings */ - const saveSettings = useCallback(async (): Promise => { + const saveSettings = useCallback(async (): Promise => { setAllQuickPrompts(updatedQuickPromptSettings); setAllSystemPrompts(updatedSystemPromptSettings); - await bulkChangeConversations(http, conversationsSettingsBulkActions, toasts); + const bulkResult = await bulkChangeConversations( + http, + conversationsSettingsBulkActions, + toasts + ); const didUpdateKnowledgeBase = knowledgeBase.isEnabledKnowledgeBase !== updatedKnowledgeBaseSettings.isEnabledKnowledgeBase; @@ -122,6 +126,8 @@ export const useSettingsUpdater = ( setKnowledgeBase(updatedKnowledgeBaseSettings); setDefaultAllow(updatedDefaultAllow); setDefaultAllowReplacement(updatedDefaultAllowReplacement); + + return bulkResult?.success ?? false; }, [ setAllQuickPrompts, updatedQuickPromptSettings, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index 3e08c145f529b..81dc563415ac8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -7,14 +7,13 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { ConversationResponse, Message, UUID } from '@kbn/elastic-assistant-common'; +import { ConversationResponse, Message } from '@kbn/elastic-assistant-common'; import { getConversation } from './get_conversation'; export interface AppendConversationMessagesParams { esClient: ElasticsearchClient; logger: Logger; conversationIndex: string; - user: { id?: UUID; name?: string }; existingConversation: ConversationResponse; messages: Message[]; } @@ -23,7 +22,6 @@ export const appendConversationMessages = async ({ esClient, logger, conversationIndex, - user, existingConversation, messages, }: AppendConversationMessagesParams): Promise => { @@ -82,7 +80,6 @@ export const appendConversationMessages = async ({ conversationIndex, id: existingConversation.id, logger, - user, }); return updatedConversation; } catch (err) { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts index 4dd02d67ee903..8a68faead82f8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.test.ts @@ -12,8 +12,16 @@ import { getCreateConversationSchemaMock, getUpdateConversationSchemaMock, } from '../__mocks__/conversations_schema.mock'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('ConversationDataWriter', () => { + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; describe('#bulk', () => { let writer: ConversationDataWriter; let esClientMock: ElasticsearchClient; @@ -39,6 +47,7 @@ describe('ConversationDataWriter', () => { ], conversationsToUpdate: [], conversationsToDelete: [], + authenticatedUser: mockUser1, }); const { docs_created: docsCreated } = (esClientMock.bulk as jest.Mock).mock.lastCall; @@ -54,6 +63,7 @@ describe('ConversationDataWriter', () => { conversationsToCreate: [getCreateConversationSchemaMock()], conversationsToUpdate: [getUpdateConversationSchemaMock()], conversationsToDelete: ['1'], + authenticatedUser: mockUser1, }); const { @@ -145,6 +155,7 @@ describe('ConversationDataWriter', () => { conversationsToCreate: [getCreateConversationSchemaMock()], conversationsToUpdate: [], conversationsToDelete: [], + authenticatedUser: mockUser1, }); expect(docsCreated.length).toEqual(1); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index 28d5bfcce398a..3797b90bf8d57 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -6,13 +6,18 @@ */ import { v4 as uuidV4 } from 'uuid'; -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + BulkOperationContainer, + BulkOperationType, + BulkResponseItem, +} from '@elastic/elasticsearch/lib/api/types'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import { ConversationCreateProps, ConversationUpdateProps, UUID, } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; import { transformToCreateScheme } from './create_conversation'; import { transformToUpdateScheme } from './update_conversation'; import { SearchEsConversationSchema } from './types'; @@ -38,6 +43,7 @@ interface BulkParams { conversationsToUpdate?: ConversationUpdateProps[]; conversationsToDelete?: string[]; isPatch?: boolean; + authenticatedUser?: AuthenticatedUser; } export interface ConversationDataWriter { @@ -71,37 +77,7 @@ export class ConversationDataWriter implements ConversationDataWriter { }); return { - errors: errors - ? items - .map((item) => - item.create?.error - ? { - message: item.create.error?.reason, - status: item.create.status, - conversation: { - id: item.create._id, - }, - } - : item.update?.error - ? { - message: item.update.error?.reason, - status: item.update.status, - conversation: { - id: item.update._id, - }, - } - : item.delete?.error - ? { - message: item.delete?.error?.reason, - status: item.delete?.status, - conversation: { - id: item.delete?._id, - }, - } - : undefined - ) - .filter((e) => e !== undefined) - : [], + errors: errors ? this.formatErrorsResponse(items) : [], docs_created: items .filter((item) => item.create?.status === 201 || item.create?.status === 200) .map((item) => item.create?._id ?? ''), @@ -132,43 +108,74 @@ export class ConversationDataWriter implements ConversationDataWriter { } }; - private buildBulkOperations = async (params: BulkParams): Promise => { - const changedAt = new Date().toISOString(); - const conversationBody = - params.conversationsToCreate?.flatMap((conversation) => [ - { create: { _index: this.options.index, _id: uuidV4() } }, - transformToCreateScheme(changedAt, this.options.spaceId, this.options.user, conversation), - ]) ?? []; - + private getUpdateConversationsQuery = async ( + conversationsToUpdate: ConversationUpdateProps[], + authenticatedUser?: AuthenticatedUser, + isPatch?: boolean + ) => { const updatedAt = new Date().toISOString(); - - const responseToUpdate = params.conversationsToUpdate - ? await this.options.esClient.search({ - body: { - query: { - ids: { - values: params.conversationsToUpdate?.map((c) => c.id), - }, + const filterByUser = authenticatedUser + ? [ + { + bool: { + should: [ + { + term: authenticatedUser.profile_uid + ? { + 'user.id': { value: authenticatedUser.profile_uid }, + } + : { + 'user.name': { value: authenticatedUser.username }, + }, + }, + ], }, }, - _source: false, - ignore_unavailable: true, - index: this.options.index, - seq_no_primary_term: true, - size: 1000, - }) - : undefined; - const conversationUpdatedBody = - params.conversationsToUpdate?.flatMap((conversation) => [ - { - update: { - _id: conversation.id, - _index: responseToUpdate?.hits.hits.find((c) => c._id === conversation.id)?._index, + ] + : []; + + const responseToUpdate = await this.options.esClient.search({ + body: { + query: { + bool: { + must: [ + { + bool: { + should: [ + { + ids: { + values: conversationsToUpdate?.map((c) => c.id), + }, + }, + ], + }, + }, + ...filterByUser, + ], }, }, - { - script: { - source: ` + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }); + + const availableConversationsToUpdate = conversationsToUpdate.filter((c) => + responseToUpdate?.hits.hits.find((ac) => ac._id === c.id) + ); + + return availableConversationsToUpdate.flatMap((conversation) => [ + { + update: { + _id: conversation.id, + _index: responseToUpdate?.hits.hits.find((c) => c._id === conversation.id)?._index, + }, + }, + { + script: { + source: ` if (params.assignEmpty == true || params.containsKey('api_config')) { if (params.assignEmpty == true || params.api_config.containsKey('connector_id')) { ctx._source.api_config.connector_id = params.api_config.connector_id; @@ -212,48 +219,151 @@ export class ConversationDataWriter implements ConversationDataWriter { } ctx._source.updated_at = params.updated_at; `, - lang: 'painless', - params: { - ...transformToUpdateScheme(updatedAt, conversation), // when assigning undefined in painless, it will remove property and wil set it to null - // for patch we don't want to remove unspecified value in payload - assignEmpty: !(params.isPatch ?? true), - }, + lang: 'painless', + params: { + ...transformToUpdateScheme(updatedAt, conversation), // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !(isPatch ?? true), }, - upsert: { counter: 1 }, }, - ]) ?? []; + upsert: { counter: 1 }, + }, + ]); + }; - const response = params.conversationsToDelete - ? await this.options.esClient.search({ - body: { - query: { - ids: { - values: params.conversationsToDelete, - }, + private getDeleteConversationsQuery = async ( + conversationsToDelete: string[], + authenticatedUser?: AuthenticatedUser + ) => { + const filterByUser = authenticatedUser + ? [ + { + bool: { + should: [ + { + term: authenticatedUser.profile_uid + ? { + 'user.id': { value: authenticatedUser.profile_uid }, + } + : { + 'user.name': { value: authenticatedUser.username }, + }, + }, + ], }, }, - _source: false, - ignore_unavailable: true, - index: this.options.index, - seq_no_primary_term: true, - size: 1000, - }) - : undefined; + ] + : []; - const conversationDeletedBody = - params.conversationsToDelete?.flatMap((conversationId) => [ + const responseToDelete = await this.options.esClient.search({ + body: { + query: { + bool: { + must: [ + { + bool: { + should: [ + { + ids: { + values: conversationsToDelete, + }, + }, + ], + }, + }, + ...filterByUser, + ], + }, + }, + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }); + + return ( + responseToDelete?.hits.hits.map((c) => [ { delete: { - _id: conversationId, - _index: response?.hits.hits.find((c) => c._id === conversationId)?._index, + _id: c._id, + _index: c._index, }, }, - ]) ?? []; + ]) ?? [] + ); + }; + + private buildBulkOperations = async (params: BulkParams): Promise => { + const changedAt = new Date().toISOString(); + const conversationCreateBody = + params.authenticatedUser && params.conversationsToCreate + ? params.conversationsToCreate.flatMap((conversation) => [ + { create: { _index: this.options.index, _id: uuidV4() } }, + transformToCreateScheme( + changedAt, + this.options.spaceId, + params.authenticatedUser as AuthenticatedUser, + conversation + ), + ]) + : []; + + const conversationDeletedBody = + params.conversationsToDelete && params.conversationsToDelete.length > 0 + ? await this.getDeleteConversationsQuery( + params.conversationsToDelete, + params.authenticatedUser + ) + : []; + + const conversationUpdatedBody = + params.conversationsToUpdate && params.conversationsToUpdate.length > 0 + ? await this.getUpdateConversationsQuery( + params.conversationsToUpdate, + params.authenticatedUser + ) + : []; return [ - ...conversationBody, + ...conversationCreateBody, ...conversationUpdatedBody, ...conversationDeletedBody, ] as BulkOperationContainer[]; }; + + private formatErrorsResponse = ( + items: Array>> + ) => { + return items + .map((item) => + item.create?.error + ? { + message: item.create.error?.reason, + status: item.create.status, + conversation: { + id: item.create._id, + }, + } + : item.update?.error + ? { + message: item.update.error?.reason, + status: item.update.status, + conversation: { + id: item.update._id, + }, + } + : item.delete?.error + ? { + message: item.delete?.error?.reason, + status: item.delete?.status, + conversation: { + id: item.delete?._id, + }, + } + : undefined + ) + .filter((e) => e !== undefined); + }; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index ede06f3462e88..ae5f4e9b53ee5 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -15,8 +15,8 @@ import { Provider, Reader, Replacement, - UUID, } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; import { getConversation } from './get_conversation'; export interface CreateMessageSchema { @@ -63,7 +63,7 @@ export interface CreateConversationParams { logger: Logger; conversationIndex: string; spaceId: string; - user: { id?: UUID; name?: string }; + user: AuthenticatedUser; conversation: ConversationCreateProps; } @@ -94,7 +94,7 @@ export const createConversation = async ({ }); return createdConversation; } catch (err) { - logger.warn(`Error creating conversation: ${err} with title: ${conversation.title}`); + logger.error(`Error creating conversation: ${err} with title: ${conversation.title}`); throw err; } }; @@ -102,7 +102,7 @@ export const createConversation = async ({ export const transformToCreateScheme = ( createdAt: string, spaceId: string, - user: { id?: UUID; name?: string }, + user: AuthenticatedUser, { title, apiConfig, @@ -115,7 +115,10 @@ export const transformToCreateScheme = ( return { '@timestamp': createdAt, created_at: createdAt, - user, + user: { + id: user.profile_uid, + name: user.username, + }, title, api_config: { connector_id: apiConfig?.connectorId, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index cee596c69cc02..371f8a8d4c3c4 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -45,10 +45,6 @@ export const getDeleteConversationOptionsMock = (): DeleteConversationParams => id: 'test', conversationIndex: '.kibana-elastic-ai-assistant-conversations', logger: loggingSystemMock.createLogger(), - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, }); describe('deleteConversation', () => { @@ -64,7 +60,7 @@ describe('deleteConversation', () => { (getConversation as unknown as jest.Mock).mockResolvedValueOnce(null); const options = getDeleteConversationOptionsMock(); const deletedConversation = await deleteConversation(options); - expect(deletedConversation).toEqual(null); + expect(deletedConversation).toEqual(undefined); }); test('Delete returns the conversation id if a conversation is returned from getConversation', async () => { @@ -72,8 +68,8 @@ describe('deleteConversation', () => { (getConversation as unknown as jest.Mock).mockResolvedValueOnce(conversation); const options = getDeleteConversationOptionsMock(); options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); - const deletedConversationId = await deleteConversation(options); - expect(deletedConversationId).toEqual(conversation.id); + const deletedConversations = await deleteConversation(options); + expect(deletedConversations).toEqual(1); }); test('Delete does not call data client if the conversation returns null', async () => { diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts index 440250d29f41b..eff39246225e1 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.ts @@ -6,42 +6,35 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { UUID } from '@kbn/elastic-assistant-common'; -import { getConversation } from './get_conversation'; export interface DeleteConversationParams { esClient: ElasticsearchClient; conversationIndex: string; id: string; logger: Logger; - user: { id?: UUID; name?: string }; } export const deleteConversation = async ({ esClient, conversationIndex, id, logger, - user, -}: DeleteConversationParams): Promise => { - const conversation = await getConversation({ esClient, conversationIndex, id, logger, user }); - if (conversation !== null) { - const response = await esClient.deleteByQuery({ - body: { - query: { - ids: { - values: [id], - }, +}: DeleteConversationParams): Promise => { + const response = await esClient.deleteByQuery({ + body: { + query: { + ids: { + values: [id], }, }, - conflicts: 'proceed', - index: conversationIndex, - refresh: true, - }); + }, + conflicts: 'proceed', + index: conversationIndex, + refresh: true, + }); - if (!response.deleted && response.deleted === 0) { - throw Error('No conversation has been deleted'); - } - return conversation.id ?? null; + if (!response.deleted && response.deleted === 0) { + logger.error(`Error deleting conversation by id: ${id}`); + throw Error('No conversation has been deleted'); } - return null; + return response.deleted; }; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index ee01db128dec6..9aadc63c926ff 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -12,6 +12,7 @@ import { estypes } from '@elastic/elasticsearch'; import { SearchEsConversationSchema } from './types'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { ConversationResponse } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; export const getConversationResponseMock = (): ConversationResponse => ({ createdAt: '2020-04-20T15:25:31.830Z', @@ -37,6 +38,14 @@ export const getConversationResponseMock = (): ConversationResponse => ({ replacements: undefined, }); +const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, +} as AuthenticatedUser; + export const getSearchConversationMock = (): estypes.SearchResponse => ({ _scroll_id: '123', @@ -104,7 +113,7 @@ describe('getConversation', () => { conversationIndex: '.kibana-elastic-ai-assistant-conversations', id: '1', logger: loggerMock, - user: { name: 'test' }, + user: mockUser1, }); const expected = getConversationResponseMock(); expect(conversation).toEqual(expected); @@ -120,7 +129,7 @@ describe('getConversation', () => { conversationIndex: '.kibana-elastic-ai-assistant-conversations', id: '1', logger: loggerMock, - user: { name: 'test' }, + user: mockUser1, }); expect(conversation).toEqual(null); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts index 7f694b27a9741..34eb69d740672 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -6,7 +6,8 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { ConversationResponse, UUID } from '@kbn/elastic-assistant-common'; +import { ConversationResponse } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin/common'; import { SearchEsConversationSchema } from './types'; import { transformESToConversations } from './transforms'; @@ -15,19 +16,52 @@ export interface GetConversationParams { logger: Logger; conversationIndex: string; id: string; - user: { id?: UUID; name?: string }; + user?: AuthenticatedUser | null; } export const getConversation = async ({ esClient, conversationIndex, id, + user, }: GetConversationParams): Promise => { + const filterByUser = user + ? [ + { + bool: { + should: [ + { + term: user.profile_uid + ? { + 'user.id': { value: user.profile_uid }, + } + : { + 'user.name': { value: user.username }, + }, + }, + ], + }, + }, + ] + : []; const response = await esClient.search({ body: { query: { - term: { - _id: id, + bool: { + must: [ + { + bool: { + should: [ + { + term: { + _id: id, + }, + }, + ], + }, + }, + ...filterByUser, + ], }, }, }, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts index a4f60c3d34314..5e72b16404043 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts @@ -116,7 +116,7 @@ describe('AIAssistantConversationsDataClient', () => { const assistantConversationsDataClient = new AIAssistantConversationsDataClient( assistantConversationsDataClientParams ); - const result = await assistantConversationsDataClient.getConversation('1'); + const result = await assistantConversationsDataClient.getConversation({ id: '1' }); expect(clusterClient.search).toHaveBeenCalledTimes(1); @@ -172,10 +172,10 @@ describe('AIAssistantConversationsDataClient', () => { assistantConversationsDataClientParams ); - await assistantConversationsDataClient.updateConversation( - getConversationMock(getQueryConversationParams()), - getUpdateConversationSchemaMock('123345') - ); + await assistantConversationsDataClient.updateConversation({ + existingConversation: getConversationMock(getQueryConversationParams()), + conversationUpdateProps: getUpdateConversationSchemaMock('12345'), + }); const params = clusterClient.updateByQuery.mock.calls[0][0] as UpdateByQueryRequest; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index a4355100af163..da3bd8ec798d5 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -108,14 +108,20 @@ export class AIAssistantConversationsDataClient { }; }; - public getConversation = async (id: string): Promise => { + public getConversation = async ({ + id, + authenticatedUser, + }: { + id: string; + authenticatedUser?: AuthenticatedUser | null; + }): Promise => { const esClient = await this.options.elasticsearchClientPromise; return getConversation({ esClient, logger: this.options.logger, conversationIndex: this.indexTemplateAndPattern.alias, id, - user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + user: authenticatedUser, }); }; @@ -126,17 +132,18 @@ export class AIAssistantConversationsDataClient { * @param options.messages Set this to true if this is a conversation that is "immutable"/"pre-packaged". * @returns The conversation updated */ - public appendConversationMessages = async ( - existingConversation: ConversationResponse, - messages: Message[] - ): Promise => { - const { currentUser } = this; + public appendConversationMessages = async ({ + existingConversation, + messages, + }: { + existingConversation: ConversationResponse; + messages: Message[]; + }): Promise => { const esClient = await this.options.elasticsearchClientPromise; return appendConversationMessages({ esClient, logger: this.options.logger, conversationIndex: this.indexTemplateAndPattern.alias, - user: { id: currentUser?.profile_uid, name: currentUser?.username }, existingConversation, messages, }); @@ -181,17 +188,20 @@ export class AIAssistantConversationsDataClient { * @param options.apiConfig Determines how uploaded conversation item values are parsed. By default, conversation items are parsed using named regex groups. See online docs for more information. * @returns The conversation created */ - public createConversation = async ( - conversation: ConversationCreateProps - ): Promise => { - const { currentUser } = this; + public createConversation = async ({ + conversation, + authenticatedUser, + }: { + conversation: ConversationCreateProps; + authenticatedUser: AuthenticatedUser; + }): Promise => { const esClient = await this.options.elasticsearchClientPromise; return createConversation({ esClient, logger: this.options.logger, conversationIndex: this.indexTemplateAndPattern.alias, spaceId: this.spaceId, - user: { id: currentUser?.profile_uid, name: currentUser?.username }, + user: authenticatedUser, conversation, }); }; @@ -208,11 +218,17 @@ export class AIAssistantConversationsDataClient { * @param options.excludeFromLastConversationStorage The new value for excludeFromLastConversationStorage, or "undefined" if this should not be updated. * @param options.replacements The new value for replacements, or "undefined" if this should not be updated. */ - public updateConversation = async ( - existingConversation: ConversationResponse, - conversationUpdateProps: ConversationUpdateProps, - isPatch?: boolean - ): Promise => { + public updateConversation = async ({ + existingConversation, + conversationUpdateProps, + authenticatedUser, + isPatch, + }: { + existingConversation: ConversationResponse; + conversationUpdateProps: ConversationUpdateProps; + authenticatedUser?: AuthenticatedUser; + isPatch?: boolean; + }): Promise => { const esClient = await this.options.elasticsearchClientPromise; return updateConversation({ esClient, @@ -221,7 +237,7 @@ export class AIAssistantConversationsDataClient { existingConversation, conversationUpdateProps, isPatch, - user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + user: authenticatedUser, }); }; @@ -233,12 +249,11 @@ export class AIAssistantConversationsDataClient { */ public deleteConversation = async (id: string) => { const esClient = await this.options.elasticsearchClientPromise; - await deleteConversation({ + return deleteConversation({ esClient, conversationIndex: this.indexTemplateAndPattern.alias, id, logger: this.options.logger, - user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, }); }; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 8b5cd8a46211f..47313d18e53a2 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -11,10 +11,10 @@ import { Replacement, Reader, ConversationUpdateProps, - UUID, Provider, MessageRole, } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin/common'; import { getConversation } from './get_conversation'; export interface UpdateConversationSchema { @@ -51,7 +51,7 @@ export interface UpdateConversationSchema { export interface UpdateConversationParams { esClient: ElasticsearchClient; logger: Logger; - user: { id?: UUID; name?: string }; + user?: AuthenticatedUser; conversationIndex: string; existingConversation: ConversationResponse; conversationUpdateProps: ConversationUpdateProps; diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts deleted file mode 100644 index 99d0215bf09ef..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { findConversationsRoute } from './find_route'; -import { getFindRequest, requestMock } from '../../__mocks__/request'; -import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND } from '@kbn/elastic-assistant-common'; -import { serverMock } from '../../__mocks__/server'; -import { requestContextMock } from '../../__mocks__/request_context'; -import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; - -describe('Find conversations route', () => { - let server: ReturnType; - let { clients, context } = requestContextMock.createTools(); - let logger: ReturnType; - - beforeEach(async () => { - server = serverMock.create(); - logger = loggingSystemMock.createLogger(); - ({ clients, context } = requestContextMock.createTools()); - - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValue( - getFindConversationsResultWithSingleHit() - ); - - findConversationsRoute(server.router, logger); - }); - - describe('status codes', () => { - test('returns 200', async () => { - const response = await server.inject( - getFindRequest(), - requestContextMock.convertContext(context) - ); - expect(response.status).toEqual(200); - }); - - test('catches error if search throws error', async () => { - clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockImplementation( - async () => { - throw new Error('Test error'); - } - ); - const response = await server.inject( - getFindRequest(), - requestContextMock.convertContext(context) - ); - expect(response.status).toEqual(500); - expect(response.body).toEqual({ - message: 'Test error', - status_code: 500, - }); - }); - }); - - describe('request validation', () => { - test('allows optional query params', async () => { - const request = requestMock.create({ - method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - query: { - page: 2, - per_page: 20, - sort_field: 'title', - fields: ['field1', 'field2'], - }, - }); - const result = server.validate(request); - - expect(result.ok).toHaveBeenCalled(); - }); - - test('disallows invalid sort fields', async () => { - const request = requestMock.create({ - method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - query: { - page: 2, - per_page: 20, - sort_field: 'name', - fields: ['field1', 'field2'], - }, - }); - const result = server.validate(request); - - expect(result.badRequest).toHaveBeenCalledWith( - `sort_field: Invalid enum value. Expected 'created_at' | 'is_default' | 'title' | 'updated_at', received 'name'` - ); - }); - - test('ignores unknown query params', async () => { - const request = requestMock.create({ - method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - query: { - invalid_value: 'test 1', - }, - }); - const result = server.validate(request); - - expect(result.ok).toHaveBeenCalled(); - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts deleted file mode 100644 index 11ab3005e987e..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_route.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IKibanaResponse, Logger } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; - -import { - ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - FindConversationsRequestQuery, - FindConversationsResponse, -} from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; - -export const findConversationsRoute = (router: ElasticAssistantPluginRouter, logger: Logger) => { - router.versioned - .get({ - access: 'public', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: { - request: { - query: buildRouteValidationWithZod(FindConversationsRequestQuery), - }, - }, - }, - async (context, request, response): Promise> => { - const assistantResponse = buildResponse(response); - try { - const { query } = request; - const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - - const result = await dataClient?.findConversations({ - perPage: query.per_page, - page: query.page, - sortField: query.sort_field, - sortOrder: query.sort_order, - filter: query.filter, - fields: query.fields, - }); - - return response.ok({ body: result }); - } catch (err) { - const error = transformError(err); - return assistantResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index bcb5de73e3c9a..88453221f5bdd 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -13,14 +13,13 @@ import { ElasticAssistantPluginSetupDependencies, GetElser, } from '../types'; -import { createConversationRoute } from './conversations/create_route'; -import { deleteConversationRoute } from './conversations/delete_route'; -import { findConversationsRoute } from './conversations/find_route'; -import { readConversationRoute } from './conversations/read_route'; -import { updateConversationRoute } from './conversations/update_route'; -import { findUserConversationsRoute } from './conversations/find_user_conversations_route'; -import { bulkActionConversationsRoute } from './conversations/bulk_actions_route'; -import { appendConversationMessageRoute } from './conversations/append_conversation_messages_route'; +import { createConversationRoute } from './user_conversations/create_route'; +import { deleteConversationRoute } from './user_conversations/delete_route'; +import { readConversationRoute } from './user_conversations/read_route'; +import { updateConversationRoute } from './user_conversations/update_route'; +import { findUserConversationsRoute } from './user_conversations/find_user_conversations_route'; +import { bulkActionConversationsRoute } from './user_conversations/bulk_actions_route'; +import { appendConversationMessageRoute } from './user_conversations/append_conversation_messages_route'; import { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; import { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; @@ -37,21 +36,20 @@ export const registerRoutes = ( logger: Logger, plugins: ElasticAssistantPluginSetupDependencies ) => { - // Conversation CRUD + // Capabilities + getCapabilitiesRoute(router); + + // User Conversations CRUD createConversationRoute(router); readConversationRoute(router); updateConversationRoute(router); deleteConversationRoute(router); appendConversationMessageRoute(router); - // Conversations bulk CRUD + // User Conversations bulk CRUD bulkActionConversationsRoute(router, logger); - // Capabilities - getCapabilitiesRoute(router); - - // Conversations search - findConversationsRoute(router, logger); + // User Conversations search findUserConversationsRoute(router); // Knowledge Base @@ -64,8 +62,10 @@ export const registerRoutes = ( ); getKnowledgeBaseStatusRoute(router, getElserId); postKnowledgeBaseRoute(router, getElserId); + // Actions Connector Execute (LLM Wrapper) postActionsConnectorExecuteRoute(router, getElserId); + // Evaluate postEvaluateRoute(router, getElserId); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.test.ts similarity index 93% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.test.ts index f60c354e6f21c..3fbb83534b1dd 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.test.ts @@ -15,10 +15,18 @@ import { getUpdateConversationSchemaMock, } from '../../__mocks__/conversations_schema.mock'; import { appendConversationMessageRoute } from './append_conversation_messages_route'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Append conversation messages route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; beforeEach(() => { server = serverMock.create(); @@ -30,6 +38,7 @@ describe('Append conversation messages route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.appendConversationMessages.mockResolvedValue( getConversationMock(getQueryConversationParams()) ); // successful append + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); appendConversationMessageRoute(server.router); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.ts similarity index 87% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.ts index 74416f91c4f64..2b9932c656ab4 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/append_conversation_messages_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/append_conversation_messages_route.ts @@ -43,7 +43,15 @@ export const appendConversationMessageRoute = (router: ElasticAssistantPluginRou try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const existingConversation = await dataClient?.getConversation(id); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } + + const existingConversation = await dataClient?.getConversation({ id, authenticatedUser }); if (existingConversation == null) { return assistantResponse.error({ body: `conversation id: "${id}" not found`, @@ -51,10 +59,10 @@ export const appendConversationMessageRoute = (router: ElasticAssistantPluginRou }); } - const conversation = await dataClient?.appendConversationMessages( + const conversation = await dataClient?.appendConversationMessages({ existingConversation, - request.body.messages - ); + messages: request.body.messages, + }); if (conversation == null) { return assistantResponse.error({ body: `conversation id: "${id}" was not updated with appended message`, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.test.ts similarity index 90% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.test.ts index 537bcb49fda06..59f48f7224930 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.test.ts @@ -11,18 +11,29 @@ import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getConversationsBulkActionRequest, requestMock } from '../../__mocks__/request'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION } from '@kbn/elastic-assistant-common'; -import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; +import { + getFindConversationsResultWithSingleHit, + getEmptyFindResult, +} from '../../__mocks__/response'; import { getCreateConversationSchemaMock, getPerformBulkActionSchemaMock, getUpdateConversationSchemaMock, } from '../../__mocks__/conversations_schema.mock'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Perform bulk action route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); let logger: ReturnType; const mockConversation = getFindConversationsResultWithSingleHit().data[0]; + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; beforeEach(async () => { server = serverMock.create(); @@ -41,11 +52,15 @@ describe('Perform bulk action route', () => { docs_deleted: [], errors: [], }); + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); bulkActionConversationsRoute(server.router, logger); }); describe('status codes', () => { it('returns 200 when performing bulk action with all dependencies present', async () => { + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValueOnce( + getEmptyFindResult() + ); const response = await server.inject( getConversationsBulkActionRequest( [getCreateConversationSchemaMock()], @@ -96,7 +111,9 @@ describe('Perform bulk action route', () => { ], total: 5, }); - + clients.elasticAssistant.getAIAssistantConversationsDataClient.findConversations.mockResolvedValueOnce( + getEmptyFindResult() + ); const response = await server.inject( getConversationsBulkActionRequest( [getCreateConversationSchemaMock()], diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.ts similarity index 83% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.ts index 39a48b1737a81..387968d540b60 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/bulk_actions_route.ts @@ -147,6 +147,32 @@ export const bulkActionConversationsRoute = ( try { const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } + + if (body.create && body.create.length > 0) { + const result = await dataClient?.findConversations({ + perPage: 100, + page: 1, + filter: `user.id:${authenticatedUser?.profile_uid} AND (${body.create + .map((c) => `title:${c.title}`) + .join(' OR ')})`, + fields: ['title'], + }); + if (result?.data != null && result.data.length > 0) { + return assistantResponse.error({ + statusCode: 409, + body: `conversations titles: "${result.data + .map((c) => c.title) + .join(',')}" already exists`, + }); + } + } const writer = await dataClient?.getWriter(); @@ -160,18 +186,19 @@ export const bulkActionConversationsRoute = ( conversationsToCreate: body.create, conversationsToDelete: body.delete?.ids, conversationsToUpdate: body.update, + authenticatedUser, }); const created = await dataClient?.findConversations({ page: 1, perPage: 1000, - filter: docsCreated.map((updatedId) => `id:${updatedId}`).join(' OR '), + filter: docsCreated.map((c) => `id:${c}`).join(' OR '), fields: ['id'], }); const updated = await dataClient?.findConversations({ page: 1, perPage: 1000, - filter: docsUpdated.map((updatedId) => `id:${updatedId}`).join(' OR '), + filter: docsUpdated.map((c) => `id:${c}`).join(' OR '), fields: ['id'], }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.test.ts similarity index 88% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.test.ts index e0a03497acf85..164d9fd05887b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.test.ts @@ -21,10 +21,18 @@ import { getQueryConversationParams, } from '../../__mocks__/conversations_schema.mock'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Create conversation route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; beforeEach(() => { server = serverMock.create(); @@ -40,6 +48,7 @@ describe('Create conversation route', () => { context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) ); + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); createConversationRoute(server.router); }); @@ -51,6 +60,15 @@ describe('Create conversation route', () => { ); expect(response.status).toEqual(200); }); + + test('returns 401 Unauthorized when request context getCurrentUser is not defined', async () => { + context.elasticAssistant.getCurrentUser.mockReturnValueOnce(null); + const response = await server.inject( + getCreateConversationRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(401); + }); }); describe('unhappy paths', () => { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts similarity index 84% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts index d987d90371cb9..8a678aa017690 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts @@ -42,12 +42,18 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const currentUser = ctx.elasticAssistant.getCurrentUser(); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } const result = await dataClient?.findConversations({ perPage: 100, page: 1, - filter: `user.id:${currentUser?.profile_uid} AND title:${request.body.title}`, + filter: `user.id:${authenticatedUser?.profile_uid} AND title:${request.body.title}`, fields: ['title'], }); if (result?.data != null && result.data.length > 0) { @@ -56,7 +62,10 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v body: `conversation title: "${request.body.title}" already exists`, }); } - const createdConversation = await dataClient?.createConversation(request.body); + const createdConversation = await dataClient?.createConversation({ + conversation: request.body, + authenticatedUser, + }); if (createdConversation == null) { return assistantResponse.error({ diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.test.ts similarity index 91% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.test.ts index 78fc56106b289..4204a7e51ef9d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.test.ts @@ -15,10 +15,18 @@ import { getConversationMock, getQueryConversationParams, } from '../../__mocks__/conversations_schema.mock'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Delete conversation route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; beforeEach(() => { server = serverMock.create(); @@ -27,6 +35,7 @@ describe('Delete conversation route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( getConversationMock(getQueryConversationParams()) ); + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); deleteConversationRoute(server.router); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.ts similarity index 87% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.ts index 77f509b3e82a5..9a6158df039b8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/delete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/delete_route.ts @@ -41,7 +41,14 @@ export const deleteConversationRoute = (router: ElasticAssistantPluginRouter) => const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const existingConversation = await dataClient?.getConversation(id); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } + const existingConversation = await dataClient?.getConversation({ id, authenticatedUser }); if (existingConversation == null) { return assistantResponse.error({ body: `conversation id: "${id}" not found`, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.test.ts similarity index 90% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.test.ts index 3025b734bc593..d25d10409420e 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.test.ts @@ -6,7 +6,7 @@ */ import { getCurrentUserFindRequest, requestMock } from '../../__mocks__/request'; -import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS } from '@kbn/elastic-assistant-common'; +import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND } from '@kbn/elastic-assistant-common'; import { serverMock } from '../../__mocks__/server'; import { requestContextMock } from '../../__mocks__/request_context'; import { getFindConversationsResultWithSingleHit } from '../../__mocks__/response'; @@ -65,7 +65,7 @@ describe('Find user conversations route', () => { test('allows optional query params', async () => { const request = requestMock.create({ method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, query: { page: 2, per_page: 20, @@ -81,7 +81,7 @@ describe('Find user conversations route', () => { test('disallows invalid sort fields', async () => { const request = requestMock.create({ method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, query: { page: 2, per_page: 20, @@ -99,7 +99,7 @@ describe('Find user conversations route', () => { test('ignores unknown query params', async () => { const request = requestMock.create({ method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, query: { invalid_value: 'test 1', }, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts similarity index 94% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts index 2a3e34ae64aef..ac522174c12bd 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts @@ -10,7 +10,7 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, } from '@kbn/elastic-assistant-common'; import { FindConversationsRequestQuery, @@ -24,7 +24,7 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) router.versioned .get({ access: 'public', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, + path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, options: { tags: ['access:elasticAssistant'], }, diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.test.ts similarity index 82% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.test.ts index d34fa7af9ea0b..36dd3045b9d23 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.test.ts @@ -14,10 +14,18 @@ import { getQueryConversationParams, } from '../../__mocks__/conversations_schema.mock'; import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Read conversation route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; const myFakeId = '99403909-ca9b-49ba-9d7a-7e5320e68d05'; beforeEach(() => { @@ -27,6 +35,7 @@ describe('Read conversation route', () => { clients.elasticAssistant.getAIAssistantConversationsDataClient.getConversation.mockResolvedValue( getConversationMock(getQueryConversationParams()) ); + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); readConversationRoute(server.router); }); @@ -63,6 +72,15 @@ describe('Read conversation route', () => { status_code: 500, }); }); + + test('returns 401 Unauthorized when request context getCurrentUser is not defined', async () => { + context.elasticAssistant.getCurrentUser.mockReturnValueOnce(null); + const response = await server.inject( + getConversationReadRequest(), + requestContextMock.convertContext(context) + ); + expect(response.status).toEqual(401); + }); }); describe('data validation', () => { diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.ts similarity index 88% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.ts index c7e2f4850cd5c..c5ef632037ceb 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/read_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/read_route.ts @@ -42,9 +42,16 @@ export const readConversationRoute = (router: ElasticAssistantPluginRouter) => { try { const ctx = await context.resolve(['core', 'elasticAssistant']); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); - const conversation = await dataClient?.getConversation(id); + const conversation = await dataClient?.getConversation({ id, authenticatedUser }); if (conversation == null) { return assistantResponse.error({ diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.test.ts similarity index 94% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.test.ts index d4c47f6302abb..2e0f25a9bec9a 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.test.ts @@ -14,10 +14,18 @@ import { getUpdateConversationSchemaMock, } from '../../__mocks__/conversations_schema.mock'; import { updateConversationRoute } from './update_route'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; describe('Update conversation route', () => { let server: ReturnType; let { clients, context } = requestContextMock.createTools(); + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; beforeEach(() => { server = serverMock.create(); @@ -30,6 +38,7 @@ describe('Update conversation route', () => { getConversationMock(getQueryConversationParams()) ); // successful update + context.elasticAssistant.getCurrentUser.mockReturnValue(mockUser1); updateConversationRoute(server.router); }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.ts similarity index 87% rename from x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts rename to x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.ts index aa64d96dc3aae..91baf71d89ac2 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/conversations/update_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/update_route.ts @@ -46,18 +46,26 @@ export const updateConversationRoute = (router: ElasticAssistantPluginRouter) => const ctx = await context.resolve(['core', 'elasticAssistant']); const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } - const existingConversation = await dataClient?.getConversation(id); + const existingConversation = await dataClient?.getConversation({ id, authenticatedUser }); if (existingConversation == null) { return assistantResponse.error({ body: `conversation id: "${id}" not found`, statusCode: 404, }); } - const conversation = await dataClient?.updateConversation( + const conversation = await dataClient?.updateConversation({ existingConversation, - request.body - ); + conversationUpdateProps: request.body, + authenticatedUser, + }); if (conversation == null) { return assistantResponse.error({ body: `conversation id: "${id}" was not updated`, diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 6fa412f40e28d..d2f9e477b7f7d 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -34,12 +34,20 @@ const ASSISTANT_TITLE = i18n.translate('xpack.securitySolution.assistant.title', defaultMessage: 'Elastic AI Assistant', }); +const LOCAL_CONVERSATIONS_MIGRATION_STATUS_TOAST_TITLE = i18n.translate( + 'xpack.securitySolution.assistant.conversationMigrationStatus.title', + { + defaultMessage: 'Local storage conversations persisted successfuly.', + } +); + /** * This component configures the Elastic AI Assistant context provider for the Security Solution app. */ export const AssistantProvider: React.FC = ({ children }) => { const { http, + notifications, storage, triggersActionsUi: { actionTypeRegistry }, docLinks: { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION }, @@ -85,19 +93,27 @@ export const AssistantProvider: React.FC = ({ children }) => { conversations as Record ).filter((c) => c.messages && c.messages.length > 0); // post bulk create - const bulkResult = await bulkChangeConversations(http, { - create: conversationsToCreate.reduce((res: Record, c) => { - res[c.id] = { ...c, title: c.id }; - return res; - }, {}), - }); + const bulkResult = await bulkChangeConversations( + http, + { + create: conversationsToCreate.reduce((res: Record, c) => { + res[c.id] = { ...c, title: c.id }; + return res; + }, {}), + }, + notifications.toasts + ); if (bulkResult && bulkResult.success) { storage.remove(`securitySolution.${LOCAL_STORAGE_KEY}`); + notifications.toasts?.addSuccess({ + iconType: 'check', + title: LOCAL_CONVERSATIONS_MIGRATION_STATUS_TOAST_TITLE, + }); } } }; migrateConversationsFromLocalStorage(); - }, [conversations, conversationsData, http, isError, isLoading, storage]); + }, [conversations, conversationsData, http, isError, isLoading, notifications.toasts, storage]); return ( Date: Wed, 7 Feb 2024 05:10:43 +0000 Subject: [PATCH 068/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/elastic_assistant/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 48b3ac39e239a..2dc7998244cd9 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -47,6 +47,7 @@ "@kbn/core-saved-objects-server", "@kbn/spaces-plugin", "@kbn/zod-helpers", + "@kbn/security-plugin-types-common", ], "exclude": [ "target/**/*", From 7bff77149ef689543dd7857bd5e37545981d01db Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 7 Feb 2024 21:52:14 -0800 Subject: [PATCH 069/141] moved anonymization to server side --- ...ost_actions_connector_execute_route.gen.ts | 56 +++--- ...ctions_connector_execute_route.schema.yaml | 95 +++++++---- .../impl/assistant/api/index.tsx | 26 +-- .../assistant/assistant_overlay/index.tsx | 6 +- .../chat_send/use_chat_send.test.tsx | 2 + .../assistant/chat_send/use_chat_send.tsx | 89 ++++------ .../impl/assistant/index.tsx | 7 +- .../impl/assistant/prompt/helpers.ts | 70 +++----- .../impl/assistant/use_conversation/index.tsx | 14 +- .../assistant/use_send_messages/index.tsx | 13 +- .../impl/assistant_context/types.tsx | 9 + .../connectorland/connector_setup/index.tsx | 3 +- .../index.test.tsx | 0 .../index.tsx | 0 .../translations.ts | 0 .../packages/kbn-elastic-assistant/index.ts | 2 +- .../elastic_assistant/server/lib/executor.ts | 21 ++- .../server/routes/helpers.ts | 29 ++++ .../routes/post_actions_connector_execute.ts | 159 +++++++++++++++++- .../public/assistant/get_comments/index.tsx | 6 +- .../public/assistant/helpers.tsx | 2 +- .../public/assistant/provider.tsx | 62 +++---- .../use_assistant_telemetry/index.tsx | 23 ++- .../timeline/tabs_content/index.tsx | 16 +- 24 files changed, 455 insertions(+), 255 deletions(-) rename x-pack/packages/kbn-elastic-assistant/impl/{new_chat_by_id => new_chat_by_title}/index.test.tsx (100%) rename x-pack/packages/kbn-elastic-assistant/impl/{new_chat_by_id => new_chat_by_title}/index.tsx (100%) rename x-pack/packages/kbn-elastic-assistant/impl/{new_chat_by_id => new_chat_by_title}/translations.ts (100%) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 4f28d619e9560..24a1b62d3471a 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -16,6 +16,38 @@ import { z } from 'zod'; * version: 1 */ +import { UUID } from '../conversations/common_attributes.gen'; + +export type RawMessageData = z.infer; +export const RawMessageData = z.object({}).catchall(z.unknown()); + +/** + * AI assistant connector execution params. + */ +export type ConnectorExecutionParams = z.infer; +export const ConnectorExecutionParams = z.object({ + subActionParams: z.object({ + messages: z.array( + z.object({ + promptText: z.string().optional(), + allow: z.array(z.string()).optional(), + allowReplacement: z.array(z.string()).optional(), + rawData: RawMessageData.optional(), + /** + * Message role. + */ + role: z.enum(['system', 'user', 'assistant']), + content: z.string(), + }) + ), + model: z.string().optional(), + n: z.number().optional(), + stop: z.array(z.string()).optional(), + temperature: z.number().optional(), + }), + subAction: z.string(), +}); + export type ExecuteConnectorRequestParams = z.infer; export const ExecuteConnectorRequestParams = z.object({ /** @@ -27,28 +59,8 @@ export type ExecuteConnectorRequestParamsInput = z.input; export const ExecuteConnectorRequestBody = z.object({ - params: z.object({ - subActionParams: z - .object({ - messages: z - .array( - z.object({ - /** - * Message role. - */ - role: z.enum(['system', 'user', 'assistant']).optional(), - content: z.string().optional(), - }) - ) - .optional(), - model: z.string().optional(), - n: z.number().optional(), - stop: z.array(z.string()).optional(), - temperature: z.number().optional(), - }) - .optional(), - subAction: z.string().optional(), - }), + conversationId: UUID, + params: ConnectorExecutionParams, alertsIndexPattern: z.string().optional(), allow: z.array(z.string()).optional(), allowReplacement: z.array(z.string()).optional(), diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index f3d1a7d5064da..4628e57d027e7 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -26,39 +26,12 @@ paths: type: object required: - params + - conversationId properties: + conversationId: + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/UUID' params: - type: object - properties: - subActionParams: - type: object - properties: - messages: - type: array - items: - type: object - properties: - role: - type: string - description: Message role. - enum: - - system - - user - - assistant - content: - type: string - model: - type: string - n: - type: number - stop: - type: array - items: - type: string - temperature: - type: number - subAction: - type: string + $ref: '#/components/schemas/ConnectorExecutionParams' alertsIndexPattern: type: string allow: @@ -118,3 +91,63 @@ paths: type: string message: type: string + +components: + schemas: + RawMessageData: + type: object + additionalProperties: true + + ConnectorExecutionParams: + type: object + description: AI assistant connector execution params. + required: + - subActionParams + - subAction + properties: + subActionParams: + type: object + required: + - messages + properties: + messages: + type: array + items: + type: object + required: + - role + - content + properties: + promptText: + type: string + allow: + type: array + items: + type: string + allowReplacement: + type: array + items: + type: string + rawData: + $ref: '#/components/schemas/RawMessageData' + role: + type: string + description: Message role. + enum: + - system + - user + - assistant + content: + type: string + model: + type: string + n: + type: number + stop: + type: array + items: + type: string + temperature: + type: number + subAction: + type: string \ No newline at end of file diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index 88b1cbe2827cc..a5491a4e9f7b1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -8,7 +8,7 @@ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/public/common'; import { HttpSetup } from '@kbn/core/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; -import type { Conversation, Message } from '../../assistant_context/types'; +import type { Conversation, RawMessage } from '../../assistant_context/types'; import { API_ERROR } from '../translations'; import { MODEL_GPT_3_5_TURBO } from '../../connectorland/models/model_selector/model_selector'; import { @@ -21,6 +21,7 @@ import { PerformEvaluationParams } from './evaluate/use_perform_evaluation'; export * from './conversations'; export interface FetchConnectorExecuteAction { + conversationId: string; isEnabledRAGAlerts: boolean; alertsIndexPattern?: string; allow?: string[]; @@ -29,11 +30,7 @@ export interface FetchConnectorExecuteAction { assistantStreamingEnabled: boolean; apiConfig: Conversation['apiConfig']; http: HttpSetup; - messages: Message[]; - onNewReplacements: ( - newReplacements: Record - ) => Promise | undefined>; - replacements?: Record; + messages: RawMessage[]; signal?: AbortSignal | undefined; size?: number; } @@ -49,6 +46,7 @@ export interface FetchConnectorExecuteResponse { } export const fetchConnectorExecuteAction = async ({ + conversationId, isEnabledRAGAlerts, alertsIndexPattern, allow, @@ -57,29 +55,22 @@ export const fetchConnectorExecuteAction = async ({ assistantStreamingEnabled, http, messages, - onNewReplacements, - replacements, apiConfig, signal, size, }: FetchConnectorExecuteAction): Promise => { - const outboundMessages = messages.map((msg) => ({ - role: msg.role, - content: msg.content, - })); - const body = apiConfig?.provider === OpenAiProviderType.OpenAi ? { model: apiConfig.model ?? MODEL_GPT_3_5_TURBO, - messages: outboundMessages, + messages, n: 1, stop: null, temperature: 0.2, } : { // Azure OpenAI and Bedrock invokeAI both expect this body format - messages: outboundMessages, + messages, }; // TODO: Remove in part 3 of streaming work for security solution @@ -92,7 +83,6 @@ export const fetchConnectorExecuteAction = async ({ alertsIndexPattern, allow, allowReplacement, - replacements, size, }); @@ -102,6 +92,7 @@ export const fetchConnectorExecuteAction = async ({ subActionParams: body, subAction: 'invokeStream', }, + conversationId, isEnabledKnowledgeBase, isEnabledRAGAlerts, ...optionalRequestParams, @@ -111,6 +102,7 @@ export const fetchConnectorExecuteAction = async ({ subActionParams: body, subAction: 'invokeAI', }, + conversationId, isEnabledKnowledgeBase, isEnabledRAGAlerts, ...optionalRequestParams, @@ -191,8 +183,6 @@ export const fetchConnectorExecuteAction = async ({ } : undefined; - await onNewReplacements(response.replacements ?? {}); - return { response: hasParsableResponse({ isEnabledRAGAlerts, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx index 6e01a277185a9..5f02af5d3175d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx @@ -44,16 +44,16 @@ export const AssistantOverlay = React.memo(() => { promptContextId: pid, conversationTitle: cTitle, }: ShowAssistantOverlayProps) => { - const newConversationId = getLastConversationTitle(cTitle); + const newConversationTitle = getLastConversationTitle(cTitle); if (so) assistantTelemetry?.reportAssistantInvoked({ - conversationId: newConversationId, + conversationId: newConversationTitle, invokedBy: 'click', }); setIsModalVisible(so); setPromptContextId(pid); - setConversationTitle(newConversationId); + setConversationTitle(newConversationTitle); }, [assistantTelemetry, getLastConversationTitle] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index 4fb3f6bf9bde4..ebb45a30f41cd 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -28,6 +28,7 @@ const removeLastMessage = jest.fn(); const appendReplacements = jest.fn(); const clearConversation = jest.fn(); const refresh = jest.fn(); +const setCurrentConversation = jest.fn(); export const testProps: UseChatSendProps = { selectedPromptContexts: {}, @@ -47,6 +48,7 @@ export const testProps: UseChatSendProps = { setSelectedPromptContexts, setUserPrompt, refresh, + setCurrentConversation, }; const robotMessage = { response: 'Response message from the robot', isError: false }; describe('use chat send', () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 81da68bd4ff16..c99bba278cfaf 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -10,7 +10,7 @@ import { HttpSetup } from '@kbn/core-http-browser'; import { SelectedPromptContext } from '../prompt_context/types'; import { useSendMessages } from '../use_send_messages'; import { useConversation } from '../use_conversation'; -import { getCombinedMessage } from '../prompt/helpers'; +import { getCombinedRawMessages } from '../prompt/helpers'; import { Conversation, Message, Prompt } from '../../..'; import { getMessageFromRawResponse } from '../helpers'; import { getDefaultSystemPrompt } from '../use_conversation/helpers'; @@ -28,6 +28,7 @@ export interface UseChatSendProps { >; setUserPrompt: React.Dispatch>; refresh: () => Promise; + setCurrentConversation: React.Dispatch>; } export interface UseChatSend { @@ -55,10 +56,10 @@ export const useChatSend = ({ setSelectedPromptContexts, setUserPrompt, refresh, + setCurrentConversation, }: UseChatSendProps): UseChatSend => { const { isLoading, sendMessages } = useSendMessages(); - const { appendMessage, appendReplacements, clearConversation, removeLastMessage } = - useConversation(); + const { clearConversation, removeLastMessage } = useConversation(); const handlePromptChange = (prompt: string) => { setPromptTextPreview(prompt); @@ -68,28 +69,31 @@ export const useChatSend = ({ // Handles sending latest user prompt to API const handleSendMessage = useCallback( async (promptText: string) => { - const onNewReplacements = (newReplacements: Record) => - appendReplacements({ - conversationId: currentConversation.id, - replacements: newReplacements, - }); - const systemPrompt = allSystemPrompts.find((prompt) => prompt.id === editingSystemPromptId); - const message = await getCombinedMessage({ + const messagesWithRawData = getCombinedRawMessages({ isNewChat: currentConversation.messages.length === 0, - currentReplacements: currentConversation.replacements, - onNewReplacements, promptText, selectedPromptContexts, selectedSystemPrompt: systemPrompt, }); - const updatedMessages = await appendMessage({ - conversationId: currentConversation.id, - message, + const updatedMessages = [ + ...currentConversation.messages, + ...messagesWithRawData.map((m) => ({ + content: `${ + currentConversation.messages.length === 0 ? `${systemPrompt?.content ?? ''}\n\n` : '' + } + ${m.promptText}`, + timestamp: new Date().toLocaleString(), + role: m.role, + })), + ]; + + setCurrentConversation({ + ...currentConversation, + messages: updatedMessages, }); - // Reset prompt context selection and preview before sending: setSelectedPromptContexts({}); setPromptTextPreview(''); @@ -97,66 +101,45 @@ export const useChatSend = ({ const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: updatedMessages ?? [], - onNewReplacements, - replacements: currentConversation.replacements ?? {}, + messages: messagesWithRawData, + conversationId: currentConversation.id, }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); - await appendMessage({ - conversationId: currentConversation.id, - message: responseMessage, + + setCurrentConversation({ + ...currentConversation, + messages: [...updatedMessages, responseMessage], }); - await refresh(); }, [ allSystemPrompts, - appendMessage, - appendReplacements, - currentConversation.apiConfig, - currentConversation.id, - currentConversation.messages.length, - currentConversation.replacements, + currentConversation, editingSystemPromptId, http, - refresh, selectedPromptContexts, sendMessages, + setCurrentConversation, setPromptTextPreview, setSelectedPromptContexts, ] ); const handleRegenerateResponse = useCallback(async () => { - const onNewReplacements = async (newReplacements: Record) => - appendReplacements({ - conversationId: currentConversation.id, - replacements: newReplacements, - }); - - const updatedMessages = await removeLastMessage(currentConversation.id); + await removeLastMessage(currentConversation.id); const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: updatedMessages ?? [], - onNewReplacements, - replacements: currentConversation.replacements ?? {}, + messages: [], + conversationId: currentConversation.id, }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); - await appendMessage({ conversationId: currentConversation.id, message: responseMessage }); - await refresh(); - }, [ - appendMessage, - appendReplacements, - currentConversation.apiConfig, - currentConversation.id, - currentConversation.replacements, - http, - refresh, - removeLastMessage, - sendMessages, - ]); + setCurrentConversation({ + ...currentConversation, + messages: [...currentConversation.messages, responseMessage], + }); + }, [currentConversation, http, removeLastMessage, sendMessages, setCurrentConversation]); const handleButtonSendMessage = useCallback( (message: string) => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 94cfd0c9d6c5e..5c7233360a964 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -166,10 +166,12 @@ const AssistantComponent: React.FC = ({ const refetchCurrentConversation = useCallback( async (cId?: string) => { - if (!cId || cId === selectedConversationTitle || !conversations[selectedConversationTitle]) { + if (cId === selectedConversationTitle || !conversations[selectedConversationTitle]) { return; } - const updatedConversation = await getConversation(cId ?? selectedConversationTitle); + const updatedConversation = await getConversation( + cId ?? conversations[selectedConversationTitle].id + ); if (updatedConversation) { setCurrentConversation(updatedConversation); } @@ -466,6 +468,7 @@ const AssistantComponent: React.FC = ({ setEditingSystemPromptId, selectedPromptContexts, setSelectedPromptContexts, + setCurrentConversation, refresh: refetchCurrentConversation, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index 3e41bcc4bb3fe..adb260e2fe993 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -5,11 +5,7 @@ * 2.0. */ -import { transformRawData } from '@kbn/elastic-assistant-common'; - -import type { Message } from '../../assistant_context/types'; -import { SYSTEM_PROMPT_CONTEXT_NON_I18N } from '../../content/prompts/system/translations'; -import { getAnonymizedValue as defaultGetAnonymizedValue } from '../get_anonymized_value'; +import type { Message, RawMessage } from '../../assistant_context/types'; import type { SelectedPromptContext } from '../prompt_context/types'; import type { Prompt } from '../types'; @@ -33,55 +29,39 @@ export const getSystemMessages = ({ return [message]; }; -export async function getCombinedMessage({ - currentReplacements, - getAnonymizedValue = defaultGetAnonymizedValue, +export function getCombinedRawMessages({ isNewChat, - onNewReplacements, promptText, selectedPromptContexts, selectedSystemPrompt, }: { - currentReplacements: Record | undefined; - getAnonymizedValue?: ({ - currentReplacements, - rawValue, - }: { - currentReplacements: Record | undefined; - rawValue: string; - }) => string; isNewChat: boolean; - onNewReplacements: ( - newReplacements: Record - ) => Promise | undefined>; promptText: string; selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; -}): Promise { - const promptContextsContent = await Promise.all( - Object.keys(selectedPromptContexts) - .sort() - .map(async (id) => { - const promptContext = await transformRawData({ - allow: selectedPromptContexts[id].allow, - allowReplacement: selectedPromptContexts[id].allowReplacement, - currentReplacements, - getAnonymizedValue, - onNewReplacements, - rawData: selectedPromptContexts[id].rawData, - }); - - return `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; - }) - ); +}): RawMessage[] { + if (Object.keys(selectedPromptContexts).length === 0) { + return [ + { + content: `${isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : ''} - return { - content: `${ - isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '' - }${promptContextsContent} + ${promptText}`, + role: 'user', // we are combining the system and user messages into one message + }, + ]; + } -${promptText}`, - role: 'user', // we are combining the system and user messages into one message - timestamp: new Date().toLocaleString(), - }; + return Object.keys(selectedPromptContexts) + .sort() + .map((id) => { + return { + content: isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '', + role: 'user', // we are combining the system and user messages into one message + timestamp: new Date().toLocaleString(), + allow: selectedPromptContexts[id].allow, + allowReplacement: selectedPromptContexts[id].allowReplacement, + rawData: selectedPromptContexts[id].rawData, + promptText, + }; + }); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 59f248a50aef5..09b12efc6d025 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -28,7 +28,8 @@ export const DEFAULT_CONVERSATION_STATE: Conversation = { }; interface AppendMessageProps { - conversationId: string; + id: string; + title: string; message: Message; } interface AmendMessageProps { @@ -52,10 +53,7 @@ interface SetApiConfigProps { } interface UseConversation { - appendMessage: ({ - conversationId, - message, - }: AppendMessageProps) => Promise; + appendMessage: ({ id, title, message }: AppendMessageProps) => Promise; amendMessage: ({ conversationId, content }: AmendMessageProps) => Promise; appendReplacements: ({ conversationId, @@ -133,9 +131,9 @@ export const useConversation = (): UseConversation => { * Append a message to the conversation[] for a given conversationId */ const appendMessage = useCallback( - async ({ conversationId, message }: AppendMessageProps): Promise => { + async ({ id, message, title }: AppendMessageProps): Promise => { assistantTelemetry?.reportAssistantMessageSent({ - conversationId, + conversationId: title, role: message.role, isEnabledKnowledgeBase, isEnabledRAGAlerts, @@ -143,7 +141,7 @@ export const useConversation = (): UseConversation => { const res = await appendConversationMessages({ http, - conversationId, + conversationId: id, messages: [message], }); return res?.messages; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx index af28a653b59e2..f406143581b21 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx @@ -9,7 +9,7 @@ import { HttpSetup } from '@kbn/core-http-browser'; import { useCallback, useState } from 'react'; import { useAssistantContext } from '../../assistant_context'; -import { Conversation, Message } from '../../assistant_context/types'; +import { Conversation, RawMessage } from '../../assistant_context/types'; import { fetchConnectorExecuteAction, FetchConnectorExecuteResponse } from '../api'; interface SendMessagesProps { @@ -17,10 +17,8 @@ interface SendMessagesProps { allowReplacement?: string[]; apiConfig: Conversation['apiConfig']; http: HttpSetup; - messages: Message[]; - onNewReplacements: ( - newReplacements: Record - ) => Promise | undefined>; + messages: RawMessage[]; + conversationId: string; replacements?: Record; } @@ -44,11 +42,12 @@ export const useSendMessages = (): UseSendMessages => { const [isLoading, setIsLoading] = useState(false); const sendMessages = useCallback( - async ({ apiConfig, http, messages, onNewReplacements, replacements }: SendMessagesProps) => { + async ({ apiConfig, http, messages, conversationId, replacements }: SendMessagesProps) => { setIsLoading(true); try { return await fetchConnectorExecuteAction({ + conversationId, isEnabledRAGAlerts: knowledgeBase.isEnabledRAGAlerts, // settings toggle alertsIndexPattern, allow: defaultAllow, @@ -57,10 +56,8 @@ export const useSendMessages = (): UseSendMessages => { isEnabledKnowledgeBase: knowledgeBase.isEnabledKnowledgeBase, assistantStreamingEnabled, http, - replacements, messages, size: knowledgeBase.latestAlerts, - onNewReplacements, }); } finally { setIsLoading(false); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 0a17f86010405..c4310ee9a54ae 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -13,6 +13,15 @@ export interface MessagePresentation { delay?: number; stream?: boolean; } + +export interface RawMessage { + role: ConversationRole; + content: string; + allow?: string[]; + allowReplacement?: string[]; + rawData?: string | Record; + promptText?: string; +} export interface Message { role: ConversationRole; reader?: ReadableStreamDefaultReader; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index cb451fb704f85..7d7a2c97c9c12 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -194,7 +194,8 @@ export const useConnectorSetup = ({ refetchConnectors?.(); setIsConnectorModalVisible(false); await appendMessage({ - conversationId: updatedConversation.id, + id: updatedConversation.id, + title: updatedConversation.title, message: { role: 'assistant', content: i18n.CONNECTOR_SETUP_COMPLETE, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/index.test.tsx similarity index 100% rename from x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.test.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/index.test.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/index.tsx similarity index 100% rename from x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/index.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/index.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/translations.ts similarity index 100% rename from x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_id/translations.ts rename to x-pack/packages/kbn-elastic-assistant/impl/new_chat_by_title/translations.ts diff --git a/x-pack/packages/kbn-elastic-assistant/index.ts b/x-pack/packages/kbn-elastic-assistant/index.ts index 5c32597135569..7f2355038170e 100644 --- a/x-pack/packages/kbn-elastic-assistant/index.ts +++ b/x-pack/packages/kbn-elastic-assistant/index.ts @@ -53,7 +53,7 @@ export { NewChat } from './impl/new_chat'; * registered where the data is available, and then the _New chat_ button can be displayed * in another part of the tree. */ -export { NewChatByTitle } from './impl/new_chat_by_id'; +export { NewChatByTitle } from './impl/new_chat_by_title'; /** * `useAssistantOverlay` is a hook that registers context with the assistant overlay, and diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index 5ccab3513ce16..9996599f9266f 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -6,14 +6,19 @@ */ import { get } from 'lodash/fp'; -import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { KibanaRequest } from '@kbn/core-http-server'; +import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { PassThrough, Readable } from 'stream'; -import { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; +import { + ConnectorExecutionParams, + ExecuteConnectorRequestBody, +} from '@kbn/elastic-assistant-common'; export interface Props { + onMessageSent: (content: string) => void; actions: ActionsPluginStart; connectorId: string; + params: ConnectorExecutionParams; request: KibanaRequest; } interface StaticResponse { @@ -23,15 +28,17 @@ interface StaticResponse { } export const executeAction = async ({ + onMessageSent, actions, - request, + params, connectorId, + request, }: Props): Promise => { const actionsClient = await actions.getActionsClientWithRequest(request); const actionResult = await actionsClient.execute({ actionId: connectorId, - params: request.body.params, + params, }); if (actionResult.status === 'error') { @@ -41,6 +48,7 @@ export const executeAction = async ({ } const content = get('data.message', actionResult); if (typeof content === 'string') { + onMessageSent(content); return { connector_id: connectorId, data: content, // the response from the actions framework @@ -48,6 +56,11 @@ export const executeAction = async ({ }; } const readable = get('data', actionResult) as Readable; + readable.read().then(({ done, value }) => { + if (done) { + onMessageSent(value); + } + }); if (typeof readable?.read !== 'function') { throw new Error('Action result status is error: result is not streamable'); diff --git a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts index a418827c4829d..715b5348acfc5 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts @@ -7,6 +7,7 @@ import { KibanaRequest } from '@kbn/core-http-server'; import { Logger } from '@kbn/core/server'; +import { Message, TraceData } from '@kbn/elastic-assistant-common'; interface GetPluginNameFromRequestParams { request: KibanaRequest; @@ -50,3 +51,31 @@ export const getPluginNameFromRequest = ({ } return defaultPluginName; }; + +export const getMessageFromRawResponse = ({ + rawContent, + isError, + traceData, +}: { + rawContent?: string; + traceData?: TraceData; + isError?: boolean; +}): Message => { + const dateTimeString = new Date().toLocaleString(); + if (rawContent) { + return { + role: 'assistant', + content: rawContent, + timestamp: dateTimeString, + isError, + traceData, + }; + } else { + return { + role: 'assistant', + content: 'Error: Response from LLM API is empty or undefined.', + timestamp: dateTimeString, + isError: true, + }; + } +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index 6c60c6c3cc698..c9718af1a4acc 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -13,6 +13,8 @@ import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, ExecuteConnectorRequestBody, Message, + getAnonymizedValue, + transformRawData, } from '@kbn/elastic-assistant-common'; import { INVOKE_ASSISTANT_ERROR_EVENT, @@ -28,9 +30,31 @@ import { buildResponse } from '../lib/build_response'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../types'; import { ESQL_RESOURCE } from './knowledge_base/constants'; import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain'; -import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from './helpers'; +import { + DEFAULT_PLUGIN_NAME, + getMessageFromRawResponse, + getPluginNameFromRequest, +} from './helpers'; import { buildRouteValidationWithZod } from './route_validation'; +export const SYSTEM_PROMPT_CONTEXT_NON_I18N = (context: string) => { + return `CONTEXT:\n"""\n${context}\n"""`; +}; + +export const getMessageContentWithReplacements = ({ + messageContent, + replacements, +}: { + messageContent: string; + replacements: Record | undefined; +}): string => + replacements != null + ? Object.keys(replacements).reduce( + (acc, replacement) => acc.replaceAll(replacement, replacements[replacement]), + messageContent + ) + : messageContent; + export const postActionsConnectorExecuteRoute = ( router: IRouter, getElser: GetElser @@ -62,6 +86,75 @@ export const postActionsConnectorExecuteRoute = ( const telemetry = assistantContext.telemetry; try { + const authenticatedUser = assistantContext.getCurrentUser(); + if (authenticatedUser == null) { + return response.unauthorized({ + body: `Authenticated user not found`, + }); + } + + const dataClient = await assistantContext.getAIAssistantConversationsDataClient(); + const conversation = await dataClient?.getConversation({ + id: request.body.conversationId, + authenticatedUser, + }); + + if (conversation == null) { + return response.notFound({ + body: `conversation id: "${request.body.conversationId}" not found`, + }); + } + + let replacements: Record | undefined; + const onNewReplacementsFunc = async (newReplacements: Record) => { + const res = await dataClient?.updateConversation({ + existingConversation: conversation, + conversationUpdateProps: { + id: request.body.conversationId, + replacements: newReplacements, + }, + }); + replacements = res?.replacements as Record | undefined; + return res?.replacements as Record | undefined; + }; + + const promptContext = await transformRawData({ + allow: request.body.params.subActionParams.messages[0].allow ?? [], + allowReplacement: + request.body.params.subActionParams.messages[0].allowReplacement ?? [], + currentReplacements: conversation.replacements as Record | undefined, + getAnonymizedValue, + onNewReplacements: onNewReplacementsFunc, + rawData: request.body.params.subActionParams.messages[0].rawData as Record< + string, + unknown[] + >, + }); + + const messageContentWithReplacements = `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; + + const anonymizedContent = `${request.body.params.subActionParams.messages[0].content}${messageContentWithReplacements} + + ${request.body.params.subActionParams.messages[0].promptText}`; + + const dateTimeString = new Date().toLocaleString(); + const updatedConversation = await dataClient?.appendConversationMessages({ + existingConversation: conversation, + messages: request.body.params.subActionParams.messages.map((m) => ({ + content: getMessageContentWithReplacements({ + messageContent: anonymizedContent, + replacements, + }), + role: m.role, + timestamp: dateTimeString, + })), + }); + if (updatedConversation == null) { + return response.badRequest({ + body: `conversation id: "${request.body.conversationId}" not updated`, + }); + } + const connectorId = decodeURIComponent(request.params.connectorId); // get the actions plugin start contract from the request context: @@ -73,7 +166,40 @@ export const postActionsConnectorExecuteRoute = ( !requestHasRequiredAnonymizationParams(request) ) { logger.debug('Executing via actions framework directly'); - const result = await executeAction({ actions, request, connectorId }); + const result = await executeAction({ + onMessageSent: (content) => { + dataClient?.appendConversationMessages({ + existingConversation: updatedConversation, + messages: [ + getMessageFromRawResponse({ + rawContent: getMessageContentWithReplacements({ + messageContent: content, + replacements, + }), + }), + ], + }); + }, + actions, + request, + connectorId, + params: { + subAction: request.body.params.subAction, + subActionParams: { + ...request.body.params.subActionParams, + messages: [ + ...(conversation.messages?.map((c) => ({ + role: c.role, + content: c.content, + })) ?? []), + { + role: request.body.params.subActionParams.messages[0].role, + content: anonymizedContent, + }, + ], + }, + }, + }); telemetry.reportEvent(INVOKE_ASSISTANT_SUCCESS_EVENT.eventType, { isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, @@ -101,9 +227,16 @@ export const postActionsConnectorExecuteRoute = ( // convert the assistant messages to LangChain messages: const langChainMessages = getLangChainMessages( - (request.body.params?.subActionParams?.messages ?? []) as Array< - Pick - > + ([ + ...(conversation.messages?.map((c) => ({ + role: c.role, + content: c.content, + })) ?? []), + { + role: request.body.params.subActionParams.messages[0].role, + content: anonymizedContent, + }, + ] ?? []) as unknown as Array> ); const elserId = await getElser(request, (await context.core).savedObjects.getClient()); @@ -137,6 +270,22 @@ export const postActionsConnectorExecuteRoute = ( isEnabledKnowledgeBase: request.body.isEnabledKnowledgeBase, isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, }); + + dataClient?.appendConversationMessages({ + existingConversation: conversation, + messages: [ + getMessageFromRawResponse({ + rawContent: langChainResponseBody.data, + traceData: langChainResponseBody.trace_data + ? { + traceId: langChainResponseBody.trace_data.trace_id, + transactionId: langChainResponseBody.trace_data.transaction_id, + } + : {}, + }), + ], + }); + return response.ok({ body: { ...langChainResponseBody, diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index 7f3d38580b1ad..4dbada304d586 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -34,11 +34,11 @@ const transformMessageWithReplacements = ({ ...message, content: showAnonymizedValues || !replacements - ? content - : getMessageContentWithReplacements({ + ? getMessageContentWithReplacements({ messageContent: content, replacements, - }), + }) + : content, }; }; diff --git a/x-pack/plugins/security_solution/public/assistant/helpers.tsx b/x-pack/plugins/security_solution/public/assistant/helpers.tsx index 821bb927d5031..7886d1c560959 100644 --- a/x-pack/plugins/security_solution/public/assistant/helpers.tsx +++ b/x-pack/plugins/security_solution/public/assistant/helpers.tsx @@ -71,7 +71,7 @@ export const getMessageContentWithReplacements = ({ }): string => replacements != null ? Object.keys(replacements).reduce( - (acc, replacement) => acc.replaceAll(replacement, replacements[replacement]), + (acc, replacement) => acc.replaceAll(replacements[replacement], replacement), messageContent ) : messageContent; diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index d2f9e477b7f7d..3e9b6b33560e0 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useCallback, useEffect } from 'react'; +import React, { useCallback } from 'react'; import { i18n } from '@kbn/i18n'; import type { IToasts } from '@kbn/core-notifications-browser'; import type { Conversation } from '@kbn/elastic-assistant'; @@ -16,6 +16,7 @@ import { } from '@kbn/elastic-assistant'; import type { FetchConversationsResponse } from '@kbn/elastic-assistant/impl/assistant/api'; +import { once } from 'lodash/fp'; import { useBasePath, useKibana } from '../common/lib/kibana'; import { useAssistantTelemetry } from './use_assistant_telemetry'; import { getComments } from './get_comments'; @@ -55,35 +56,15 @@ export const AssistantProvider: React.FC = ({ children }) => { const basePath = useBasePath(); const baseConversations = useBaseConversations(); - const onFetchedConversations = useCallback( - (conversationsData: FetchConversationsResponse): Record => - mergeBaseWithPersistedConversations({}, conversationsData), - [] - ); - const { - data: conversationsData, - isLoading, - isError, - } = useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); const assistantAvailability = useAssistantAvailability(); const assistantTelemetry = useAssistantTelemetry(); - const { defaultAllow, defaultAllowReplacement, setDefaultAllow, setDefaultAllowReplacement } = - useAnonymizationStore(); - - const { signalIndexName } = useSignalIndex(); - const alertsIndexPattern = signalIndexName ?? undefined; - const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) - - // migrate conversations with messages from the local storage - // won't happen again if the user conversations exist in the index - const conversations = storage.get(`securitySolution.${LOCAL_STORAGE_KEY}`); - - useEffect(() => { - const migrateConversationsFromLocalStorage = async () => { + const migrateConversationsFromLocalStorage = once( + async (conversationsData: Record) => { + // migrate conversations with messages from the local storage + // won't happen next time + const conversations = storage.get(`securitySolution.${LOCAL_STORAGE_KEY}`); if ( - !isLoading && - !isError && conversationsData && Object.keys(conversationsData).length === 0 && conversations && @@ -109,11 +90,34 @@ export const AssistantProvider: React.FC = ({ children }) => { iconType: 'check', title: LOCAL_CONVERSATIONS_MIGRATION_STATUS_TOAST_TITLE, }); + return true; } + return false; } - }; - migrateConversationsFromLocalStorage(); - }, [conversations, conversationsData, http, isError, isLoading, notifications.toasts, storage]); + } + ); + const onFetchedConversations = useCallback( + (conversationsData: FetchConversationsResponse): Record => { + const mergedData = mergeBaseWithPersistedConversations({}, conversationsData); + if (assistantAvailability.isAssistantEnabled && assistantAvailability.hasAssistantPrivilege) { + migrateConversationsFromLocalStorage(mergedData); + } + return mergedData; + }, + [ + assistantAvailability.hasAssistantPrivilege, + assistantAvailability.isAssistantEnabled, + migrateConversationsFromLocalStorage, + ] + ); + useFetchCurrentUserConversations({ http, onFetch: onFetchedConversations }); + + const { defaultAllow, defaultAllowReplacement, setDefaultAllow, setDefaultAllowReplacement } = + useAnonymizationStore(); + + const { signalIndexName } = useSignalIndex(); + const alertsIndexPattern = signalIndexName ?? undefined; + const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) return ( { const { - services: { telemetry, http }, + services: { telemetry }, } = useKibana(); const baseConversations = useBaseConversations(); - const getAnonymizedConversationId = useCallback( - async (id) => { - const conversation = baseConversations[id] - ? baseConversations[id] - : await getConversationById({ http, id }); - const convo = (conversation as Conversation) ?? { isDefault: false }; - return convo.isDefault ? id : 'Custom'; + const getAnonymizedConversationTitle = useCallback( + async (title) => { + // With persistent storage for conversation replacing id to title, because id is UUID now + // and doesn't make any value for telemetry tracking + return baseConversations[title] ? title : 'Custom'; }, - [baseConversations, http] + [baseConversations] ); const reportTelemetry = useCallback( @@ -38,9 +35,9 @@ export const useAssistantTelemetry = (): AssistantTelemetry => { }> => fn({ ...rest, - conversationId: await getAnonymizedConversationId(conversationId), + conversationId: await getAnonymizedConversationTitle(conversationId), }), - [getAnonymizedConversationId] + [getAnonymizedConversationTitle] ); return { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index 9ae13a8364f69..d00aec7f85e4e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -98,7 +98,7 @@ interface BasicTimelineTab { type ActiveTimelineTabProps = BasicTimelineTab & { activeTimelineTab: TimelineTabs; showTimeline: boolean; - setConversationId: Dispatch>; + setConversationTitle: Dispatch>; }; const ActiveTimelineTab = memo( @@ -108,7 +108,7 @@ const ActiveTimelineTab = memo( rowRenderers, timelineId, timelineType, - setConversationId, + setConversationTitle, showTimeline, }) => { const { hasAssistantPrivilege } = useAssistantAvailability(); @@ -149,7 +149,7 @@ const ActiveTimelineTab = memo( > {activeTimelineTab === TimelineTabs.securityAssistant ? ( ( } else { return null; } - }, [activeTimelineTab, setConversationId, showTimeline]); + }, [activeTimelineTab, setConversationTitle, showTimeline]); /* Future developer -> why are we doing that * It is really expansive to re-render the QueryTab because the drag/drop @@ -295,7 +295,7 @@ const TabsContentComponent: React.FC = ({ const isEnterprisePlus = useLicense().isEnterprise(); - const [conversationId, setConversationId] = useState(TIMELINE_CONVERSATION_TITLE); + const [conversationTitle, setConversationTitle] = useState(TIMELINE_CONVERSATION_TITLE); const { reportAssistantInvoked } = useAssistantTelemetry(); const allTimelineNoteIds = useMemo(() => { @@ -348,11 +348,11 @@ const TabsContentComponent: React.FC = ({ setActiveTab(TimelineTabs.securityAssistant); if (activeTab !== TimelineTabs.securityAssistant) { reportAssistantInvoked({ - conversationId, + conversationId: conversationTitle, invokedBy: TIMELINE_CONVERSATION_TITLE, }); } - }, [activeTab, conversationId, reportAssistantInvoked, setActiveTab]); + }, [activeTab, conversationTitle, reportAssistantInvoked, setActiveTab]); const setEsqlAsActiveTab = useCallback(() => { dispatch( @@ -486,7 +486,7 @@ const TabsContentComponent: React.FC = ({ timelineId={timelineId} timelineType={timelineType} timelineDescription={timelineDescription} - setConversationId={setConversationId} + setConversationTitle={setConversationTitle} showTimeline={showTimeline} /> From 9f3c8ae506f6e9a60e7445165429f4c2ace3a0c2 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Fri, 9 Feb 2024 15:16:11 -0800 Subject: [PATCH 070/141] fixed replacements --- .../impl/data_anonymization/helpers/index.ts | 14 +++ .../transform_raw_data/index.test.tsx | 16 +-- .../transform_raw_data/index.tsx | 10 +- ...ost_actions_connector_execute_route.gen.ts | 4 - ...ctions_connector_execute_route.schema.yaml | 12 -- .../kbn-elastic-assistant-common/index.ts | 4 + .../impl/assistant/api/index.test.tsx | 52 +-------- .../impl/assistant/api/index.tsx | 78 +++---------- .../assistant/chat_send/use_chat_send.tsx | 41 ++++--- .../impl/assistant/helpers.ts | 4 - .../impl/assistant/prompt/helpers.ts | 64 +++++++---- .../assistant/use_send_messages/index.tsx | 5 +- .../impl/assistant_context/types.tsx | 8 -- .../routes/post_actions_connector_execute.ts | 105 +++++++----------- 14 files changed, 156 insertions(+), 261 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/helpers/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/helpers/index.ts index 410ef5fc8cc48..e048275e3d671 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/helpers/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/helpers/index.ts @@ -21,3 +21,17 @@ export const isAnonymized = ({ allowReplacementSet: Set; field: string; }): boolean => allowReplacementSet.has(field); + +export const getMessageContentWithoutReplacements = ({ + messageContent, + replacements, +}: { + messageContent: string; + replacements: Record | undefined; +}): string => + replacements != null + ? Object.keys(replacements).reduce( + (acc, replacement) => acc.replaceAll(replacement, replacements[replacement]), + messageContent + ) + : messageContent; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx index 1484913c1e37b..ded518deece66 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx @@ -9,7 +9,7 @@ import { mockGetAnonymizedValue } from '../../mock/get_anonymized_value'; import { transformRawData } from '.'; describe('transformRawData', () => { - it('returns non-anonymized data when rawData is a string', async () => { + it('returns non-anonymized data when rawData is a string', () => { const inputRawData = { allow: ['field1'], allowReplacement: ['field1', 'field2'], @@ -17,7 +17,7 @@ describe('transformRawData', () => { rawData: 'this will not be anonymized', }; - const result = await transformRawData({ + const result = transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -29,7 +29,7 @@ describe('transformRawData', () => { expect(result).toEqual('this will not be anonymized'); }); - it('calls onNewReplacements with the expected replacements', async () => { + it('calls onNewReplacements with the expected replacements', () => { const inputRawData = { allow: ['field1'], allowReplacement: ['field1'], @@ -39,7 +39,7 @@ describe('transformRawData', () => { const onNewReplacements = jest.fn(); - await transformRawData({ + transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -51,7 +51,7 @@ describe('transformRawData', () => { expect(onNewReplacements).toHaveBeenCalledWith({ '1eulav': 'value1' }); }); - it('returns the expected mix of anonymized and non-anonymized data as a CSV string', async () => { + it('returns the expected mix of anonymized and non-anonymized data as a CSV string', () => { const inputRawData = { allow: ['field1', 'field2'], allowReplacement: ['field1'], // only field 1 will be anonymized @@ -59,7 +59,7 @@ describe('transformRawData', () => { rawData: { field1: ['value1', 'value2'], field2: ['value3', 'value4'] }, }; - const result = await transformRawData({ + const result = transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, @@ -71,7 +71,7 @@ describe('transformRawData', () => { expect(result).toEqual('field1,1eulav,2eulav\nfield2,value3,value4'); // only field 1 is anonymized }); - it('omits fields that are not included in the `allow` list, even if they are members of `allowReplacement`', async () => { + it('omits fields that are not included in the `allow` list, even if they are members of `allowReplacement`', () => { const inputRawData = { allow: ['field1', 'field2'], // field3 is NOT allowed allowReplacement: ['field1', 'field3'], // field3 is requested to be anonymized @@ -83,7 +83,7 @@ describe('transformRawData', () => { }, }; - const result = await transformRawData({ + const result = transformRawData({ allow: inputRawData.allow, allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx index 5c621304d7c8e..5846e054fbab7 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx @@ -9,7 +9,7 @@ import { getAnonymizedData } from '../get_anonymized_data'; import { getAnonymizedValues } from '../get_anonymized_values'; import { getCsvFromData } from '../get_csv_from_data'; -export const transformRawData = async ({ +export const transformRawData = ({ allow, allowReplacement, currentReplacements, @@ -27,11 +27,9 @@ export const transformRawData = async ({ currentReplacements: Record | undefined; rawValue: string; }) => string; - onNewReplacements: ( - replacements: Record - ) => Promise | undefined>; + onNewReplacements: (replacements: Record) => Record | undefined; rawData: string | Record; -}): Promise => { +}): string => { if (typeof rawData === 'string') { return rawData; } @@ -46,7 +44,7 @@ export const transformRawData = async ({ }); if (onNewReplacements != null) { - await onNewReplacements(anonymizedData.replacements); + onNewReplacements(anonymizedData.replacements); } return getCsvFromData(anonymizedData.anonymizedData); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 24a1b62d3471a..8a052f0fdd770 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -29,10 +29,6 @@ export const ConnectorExecutionParams = z.object({ subActionParams: z.object({ messages: z.array( z.object({ - promptText: z.string().optional(), - allow: z.array(z.string()).optional(), - allowReplacement: z.array(z.string()).optional(), - rawData: RawMessageData.optional(), /** * Message role. */ diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index 4628e57d027e7..059ce13304f75 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -118,18 +118,6 @@ components: - role - content properties: - promptText: - type: string - allow: - type: array - items: - type: string - allowReplacement: - type: array - items: - type: string - rawData: - $ref: '#/components/schemas/RawMessageData' role: type: string description: Message role. diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts index dc4fcef1d66ee..19f25cdeb8daa 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts @@ -21,5 +21,9 @@ export { } from './impl/data_anonymization/helpers'; export { transformRawData } from './impl/data_anonymization/transform_raw_data'; +export { getAnonymizedData } from './impl/data_anonymization/get_anonymized_data'; +export { getAnonymizedValues } from './impl/data_anonymization/get_anonymized_values'; +export { getCsvFromData } from './impl/data_anonymization/get_csv_from_data'; +export { getMessageContentWithoutReplacements } from './impl/data_anonymization/helpers'; export * from './constants'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx index 79a7cb91c37de..d1efa0b032d6a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx @@ -13,7 +13,6 @@ import { fetchConnectorExecuteAction, FetchConnectorExecuteAction, getKnowledgeBaseStatus, - postEvaluation, postKnowledgeBase, } from '.'; import type { Conversation, Message } from '../../assistant_context/types'; @@ -41,7 +40,7 @@ const fetchConnectorArgs: FetchConnectorExecuteAction = { assistantStreamingEnabled: true, http: mockHttp, messages, - onNewReplacements: jest.fn(), + conversationId: 'test', }; describe('API tests', () => { beforeEach(() => { @@ -348,53 +347,4 @@ describe('API tests', () => { await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); }); }); - - describe('postEvaluation', () => { - it('calls the knowledge base API when correct resource path', async () => { - (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true }); - const testProps = { - http: mockHttp, - evalParams: { - agents: ['not', 'alphabetical'], - dataset: '{}', - datasetName: 'Test Dataset', - projectName: 'Test Project Name', - runName: 'Test Run Name', - evalModel: ['not', 'alphabetical'], - evalPrompt: 'evalPrompt', - evaluationType: ['not', 'alphabetical'], - models: ['not', 'alphabetical'], - outputIndex: 'outputIndex', - }, - }; - - await postEvaluation(testProps); - - expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', { - method: 'POST', - body: '{"dataset":{},"evalPrompt":"evalPrompt"}', - headers: { 'Content-Type': 'application/json' }, - query: { - models: 'alphabetical,not', - agents: 'alphabetical,not', - datasetName: 'Test Dataset', - evaluationType: 'alphabetical,not', - evalModel: 'alphabetical,not', - outputIndex: 'outputIndex', - projectName: 'Test Project Name', - runName: 'Test Run Name', - }, - signal: undefined, - version: '1', - }); - }); - it('returns error when error is an error', async () => { - const error = 'simulated error'; - (mockHttp.fetch as jest.Mock).mockImplementation(() => { - throw new Error(error); - }); - - await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); - }); - }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index a5491a4e9f7b1..939e9f0933bb8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -8,7 +8,7 @@ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/public/common'; import { HttpSetup } from '@kbn/core/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; -import type { Conversation, RawMessage } from '../../assistant_context/types'; +import type { Conversation, Message } from '../../assistant_context/types'; import { API_ERROR } from '../translations'; import { MODEL_GPT_3_5_TURBO } from '../../connectorland/models/model_selector/model_selector'; import { @@ -16,8 +16,6 @@ import { getOptionalRequestParams, hasParsableResponse, } from '../helpers'; -import { PerformEvaluationParams } from './evaluate/use_perform_evaluation'; - export * from './conversations'; export interface FetchConnectorExecuteAction { @@ -30,7 +28,8 @@ export interface FetchConnectorExecuteAction { assistantStreamingEnabled: boolean; apiConfig: Conversation['apiConfig']; http: HttpSetup; - messages: RawMessage[]; + messages: Message[]; + replacements?: Record; signal?: AbortSignal | undefined; size?: number; } @@ -55,22 +54,28 @@ export const fetchConnectorExecuteAction = async ({ assistantStreamingEnabled, http, messages, + replacements, apiConfig, signal, size, }: FetchConnectorExecuteAction): Promise => { + const outboundMessages = messages.map((msg) => ({ + role: msg.role, + content: msg.content, + })); + const body = apiConfig?.provider === OpenAiProviderType.OpenAi ? { model: apiConfig.model ?? MODEL_GPT_3_5_TURBO, - messages, + messages: outboundMessages, n: 1, stop: null, temperature: 0.2, } : { // Azure OpenAI and Bedrock invokeAI both expect this body format - messages, + messages: outboundMessages, }; // TODO: Remove in part 3 of streaming work for security solution @@ -93,6 +98,7 @@ export const fetchConnectorExecuteAction = async ({ subAction: 'invokeStream', }, conversationId, + replacements, isEnabledKnowledgeBase, isEnabledRAGAlerts, ...optionalRequestParams, @@ -103,6 +109,7 @@ export const fetchConnectorExecuteAction = async ({ subAction: 'invokeAI', }, conversationId, + replacements, isEnabledKnowledgeBase, isEnabledRAGAlerts, ...optionalRequestParams, @@ -334,62 +341,3 @@ export const deleteKnowledgeBase = async ({ return error as IHttpFetchError; } }; - -export interface PostEvaluationParams { - http: HttpSetup; - evalParams?: PerformEvaluationParams; - signal?: AbortSignal | undefined; -} - -export interface PostEvaluationResponse { - evaluationId: string; - success: boolean; -} - -/** - * API call for evaluating models. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.evalParams] - Params necessary for evaluation - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const postEvaluation = async ({ - http, - evalParams, - signal, -}: PostEvaluationParams): Promise => { - try { - const path = `/internal/elastic_assistant/evaluate`; - const query = { - agents: evalParams?.agents.sort()?.join(','), - datasetName: evalParams?.datasetName, - evaluationType: evalParams?.evaluationType.sort()?.join(','), - evalModel: evalParams?.evalModel.sort()?.join(','), - outputIndex: evalParams?.outputIndex, - models: evalParams?.models.sort()?.join(','), - projectName: evalParams?.projectName, - runName: evalParams?.runName, - }; - - const response = await http.fetch(path, { - method: 'POST', - body: JSON.stringify({ - dataset: JSON.parse(evalParams?.dataset ?? '[]'), - evalPrompt: evalParams?.evalPrompt ?? '', - }), - headers: { - 'Content-Type': 'application/json', - }, - query, - signal, - version: '1', - }); - - return response as PostEvaluationResponse; - } catch (error) { - return error as IHttpFetchError; - } -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index c99bba278cfaf..2256fc0239f36 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -7,10 +7,11 @@ import React, { useCallback } from 'react'; import { HttpSetup } from '@kbn/core-http-browser'; +import { getMessageContentWithoutReplacements } from '@kbn/elastic-assistant-common'; import { SelectedPromptContext } from '../prompt_context/types'; import { useSendMessages } from '../use_send_messages'; import { useConversation } from '../use_conversation'; -import { getCombinedRawMessages } from '../prompt/helpers'; +import { getCombinedMessage } from '../prompt/helpers'; import { Conversation, Message, Prompt } from '../../..'; import { getMessageFromRawResponse } from '../helpers'; import { getDefaultSystemPrompt } from '../use_conversation/helpers'; @@ -69,31 +70,38 @@ export const useChatSend = ({ // Handles sending latest user prompt to API const handleSendMessage = useCallback( async (promptText: string) => { + let replacements: Record | undefined; + const onNewReplacements = (newReplacements: Record) => { + replacements = { ...(currentConversation.replacements ?? {}), ...newReplacements }; + setCurrentConversation({ + ...currentConversation, + replacements, + }); + }; + const systemPrompt = allSystemPrompts.find((prompt) => prompt.id === editingSystemPromptId); - const messagesWithRawData = getCombinedRawMessages({ + const userMessages = getCombinedMessage({ isNewChat: currentConversation.messages.length === 0, + currentReplacements: currentConversation.replacements, promptText, selectedPromptContexts, selectedSystemPrompt: systemPrompt, + onNewReplacements, }); - const updatedMessages = [ - ...currentConversation.messages, - ...messagesWithRawData.map((m) => ({ - content: `${ - currentConversation.messages.length === 0 ? `${systemPrompt?.content ?? ''}\n\n` : '' - } - ${m.promptText}`, - timestamp: new Date().toLocaleString(), - role: m.role, - })), - ]; - + const updatedMessages = [...currentConversation.messages, ...userMessages]; setCurrentConversation({ ...currentConversation, - messages: updatedMessages, + messages: updatedMessages.map((m) => ({ + ...m, + content: getMessageContentWithoutReplacements({ + messageContent: m.content ?? '', + replacements, + }), + })), }); + // Reset prompt context selection and preview before sending: setSelectedPromptContexts({}); setPromptTextPreview(''); @@ -101,8 +109,9 @@ export const useChatSend = ({ const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: messagesWithRawData, + messages: userMessages, conversationId: currentConversation.id, + replacements, }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index fe506e0b7eb79..b00dd3010418f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -116,20 +116,17 @@ export const getOptionalRequestParams = ({ alertsIndexPattern, allow, allowReplacement, - replacements, size, }: { isEnabledRAGAlerts: boolean; alertsIndexPattern?: string; allow?: string[]; allowReplacement?: string[]; - replacements?: Record; size?: number; }): OptionalRequestParams => { const optionalAlertsIndexPattern = alertsIndexPattern ? { alertsIndexPattern } : undefined; const optionalAllow = allow ? { allow } : undefined; const optionalAllowReplacement = allowReplacement ? { allowReplacement } : undefined; - const optionalReplacements = replacements ? { replacements } : undefined; const optionalSize = size ? { size } : undefined; // the settings toggle must be enabled: @@ -141,7 +138,6 @@ export const getOptionalRequestParams = ({ ...optionalAlertsIndexPattern, ...optionalAllow, ...optionalAllowReplacement, - ...optionalReplacements, ...optionalSize, }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index adb260e2fe993..49ff9398708b3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -5,9 +5,16 @@ * 2.0. */ -import type { Message, RawMessage } from '../../assistant_context/types'; +import { + getAnonymizedData, + getAnonymizedValues, + getCsvFromData, +} from '@kbn/elastic-assistant-common'; +import { getAnonymizedValue as defaultGetAnonymizedValue } from '../get_anonymized_value'; +import type { Message } from '../../assistant_context/types'; import type { SelectedPromptContext } from '../prompt_context/types'; import type { Prompt } from '../types'; +import { SYSTEM_PROMPT_CONTEXT_NON_I18N } from '../../content/prompts/system/translations'; export const getSystemMessages = ({ isNewChat, @@ -29,39 +36,58 @@ export const getSystemMessages = ({ return [message]; }; -export function getCombinedRawMessages({ +export function getCombinedMessage({ + currentReplacements, + getAnonymizedValue = defaultGetAnonymizedValue, + onNewReplacements, isNewChat, promptText, selectedPromptContexts, selectedSystemPrompt, }: { + currentReplacements: Record | undefined; + getAnonymizedValue?: ({ + currentReplacements, + rawValue, + }: { + currentReplacements: Record | undefined; + rawValue: string; + }) => string; isNewChat: boolean; + onNewReplacements: (newReplacements: Record) => void; promptText: string; selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; -}): RawMessage[] { - if (Object.keys(selectedPromptContexts).length === 0) { - return [ - { - content: `${isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : ''} - - ${promptText}`, - role: 'user', // we are combining the system and user messages into one message - }, - ]; - } - +}): Message[] { return Object.keys(selectedPromptContexts) .sort() .map((id) => { + let content: string; + if (typeof selectedPromptContexts[id].rawData === 'string') { + content = `${SYSTEM_PROMPT_CONTEXT_NON_I18N( + selectedPromptContexts[id].rawData.toString() + )}`; + } else { + const anonymizedData = getAnonymizedData({ + allow: selectedPromptContexts[id].allow, + allowReplacement: selectedPromptContexts[id].allowReplacement, + currentReplacements, + rawData: selectedPromptContexts[id].rawData as Record, + getAnonymizedValue, + getAnonymizedValues, + }); + + const promptContext = getCsvFromData(anonymizedData.anonymizedData); + content = `${ + isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '' + }${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)} + ${promptText}`; + onNewReplacements(anonymizedData.replacements); + } return { - content: isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '', + content, role: 'user', // we are combining the system and user messages into one message timestamp: new Date().toLocaleString(), - allow: selectedPromptContexts[id].allow, - allowReplacement: selectedPromptContexts[id].allowReplacement, - rawData: selectedPromptContexts[id].rawData, - promptText, }; }); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx index f406143581b21..9d89eec3622c1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx @@ -9,7 +9,7 @@ import { HttpSetup } from '@kbn/core-http-browser'; import { useCallback, useState } from 'react'; import { useAssistantContext } from '../../assistant_context'; -import { Conversation, RawMessage } from '../../assistant_context/types'; +import { Conversation, Message } from '../../assistant_context/types'; import { fetchConnectorExecuteAction, FetchConnectorExecuteResponse } from '../api'; interface SendMessagesProps { @@ -17,7 +17,7 @@ interface SendMessagesProps { allowReplacement?: string[]; apiConfig: Conversation['apiConfig']; http: HttpSetup; - messages: RawMessage[]; + messages: Message[]; conversationId: string; replacements?: Record; } @@ -57,6 +57,7 @@ export const useSendMessages = (): UseSendMessages => { assistantStreamingEnabled, http, messages, + replacements, size: knowledgeBase.latestAlerts, }); } finally { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index c4310ee9a54ae..0fd7f187b02a4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -14,14 +14,6 @@ export interface MessagePresentation { stream?: boolean; } -export interface RawMessage { - role: ConversationRole; - content: string; - allow?: string[]; - allowReplacement?: string[]; - rawData?: string | Record; - promptText?: string; -} export interface Message { role: ConversationRole; reader?: ReadableStreamDefaultReader; diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index c9718af1a4acc..a183774f9fe47 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -13,8 +13,7 @@ import { ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, ExecuteConnectorRequestBody, Message, - getAnonymizedValue, - transformRawData, + getMessageContentWithoutReplacements, } from '@kbn/elastic-assistant-common'; import { INVOKE_ASSISTANT_ERROR_EVENT, @@ -37,24 +36,6 @@ import { } from './helpers'; import { buildRouteValidationWithZod } from './route_validation'; -export const SYSTEM_PROMPT_CONTEXT_NON_I18N = (context: string) => { - return `CONTEXT:\n"""\n${context}\n"""`; -}; - -export const getMessageContentWithReplacements = ({ - messageContent, - replacements, -}: { - messageContent: string; - replacements: Record | undefined; -}): string => - replacements != null - ? Object.keys(replacements).reduce( - (acc, replacement) => acc.replaceAll(replacement, replacements[replacement]), - messageContent - ) - : messageContent; - export const postActionsConnectorExecuteRoute = ( router: IRouter, getElser: GetElser @@ -105,53 +86,49 @@ export const postActionsConnectorExecuteRoute = ( }); } - let replacements: Record | undefined; - const onNewReplacementsFunc = async (newReplacements: Record) => { - const res = await dataClient?.updateConversation({ + if (request.body.replacements) { + await dataClient?.updateConversation({ existingConversation: conversation, conversationUpdateProps: { id: request.body.conversationId, - replacements: newReplacements, + replacements: request.body.replacements, }, }); - replacements = res?.replacements as Record | undefined; - return res?.replacements as Record | undefined; - }; - - const promptContext = await transformRawData({ - allow: request.body.params.subActionParams.messages[0].allow ?? [], - allowReplacement: - request.body.params.subActionParams.messages[0].allowReplacement ?? [], - currentReplacements: conversation.replacements as Record | undefined, - getAnonymizedValue, - onNewReplacements: onNewReplacementsFunc, - rawData: request.body.params.subActionParams.messages[0].rawData as Record< - string, - unknown[] - >, - }); + } - const messageContentWithReplacements = `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; + const dateTimeString = new Date().toLocaleString(); - const anonymizedContent = `${request.body.params.subActionParams.messages[0].content}${messageContentWithReplacements} + const appendMessageFuncs = request.body.params.subActionParams.messages.map( + (userMessage) => async () => { + const res = await dataClient?.appendConversationMessages({ + existingConversation: conversation, + messages: request.body.params.subActionParams.messages.map((m) => ({ + content: getMessageContentWithoutReplacements({ + messageContent: userMessage.content, + replacements: request.body.replacements as Record | undefined, + }), + role: m.role, + timestamp: dateTimeString, + })), + }); + if (res == null) { + return response.badRequest({ + body: `conversation id: "${request.body.conversationId}" not updated`, + }); + } + } + ); - ${request.body.params.subActionParams.messages[0].promptText}`; + await Promise.all(appendMessageFuncs.map((appendMessageFunc) => appendMessageFunc())); - const dateTimeString = new Date().toLocaleString(); - const updatedConversation = await dataClient?.appendConversationMessages({ - existingConversation: conversation, - messages: request.body.params.subActionParams.messages.map((m) => ({ - content: getMessageContentWithReplacements({ - messageContent: anonymizedContent, - replacements, - }), - role: m.role, - timestamp: dateTimeString, - })), + const updatedConversation = await dataClient?.getConversation({ + id: request.body.conversationId, + authenticatedUser, }); + if (updatedConversation == null) { - return response.badRequest({ - body: `conversation id: "${request.body.conversationId}" not updated`, + return response.notFound({ + body: `conversation id: "${request.body.conversationId}" not found`, }); } @@ -172,9 +149,11 @@ export const postActionsConnectorExecuteRoute = ( existingConversation: updatedConversation, messages: [ getMessageFromRawResponse({ - rawContent: getMessageContentWithReplacements({ + rawContent: getMessageContentWithoutReplacements({ messageContent: content, - replacements, + replacements: request.body.replacements as + | Record + | undefined, }), }), ], @@ -192,10 +171,7 @@ export const postActionsConnectorExecuteRoute = ( role: c.role, content: c.content, })) ?? []), - { - role: request.body.params.subActionParams.messages[0].role, - content: anonymizedContent, - }, + ...request.body.params.subActionParams.messages, ], }, }, @@ -232,10 +208,7 @@ export const postActionsConnectorExecuteRoute = ( role: c.role, content: c.content, })) ?? []), - { - role: request.body.params.subActionParams.messages[0].role, - content: anonymizedContent, - }, + request.body.params.subActionParams.messages, ] ?? []) as unknown as Array> ); From f0df29f66ca0100bdb530c864a4f09d429432e0d Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Fri, 9 Feb 2024 17:18:31 -0800 Subject: [PATCH 071/141] fixed no replacements message --- .../kbn-elastic-assistant-common/index.ts | 3 - .../assistant/chat_send/use_chat_send.tsx | 21 +++---- .../impl/assistant/prompt/helpers.ts | 58 ++++++++----------- 3 files changed, 35 insertions(+), 47 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts index 19f25cdeb8daa..14b462f0d7b4d 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts @@ -21,9 +21,6 @@ export { } from './impl/data_anonymization/helpers'; export { transformRawData } from './impl/data_anonymization/transform_raw_data'; -export { getAnonymizedData } from './impl/data_anonymization/get_anonymized_data'; -export { getAnonymizedValues } from './impl/data_anonymization/get_anonymized_values'; -export { getCsvFromData } from './impl/data_anonymization/get_csv_from_data'; export { getMessageContentWithoutReplacements } from './impl/data_anonymization/helpers'; export * from './constants'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 2256fc0239f36..f53b3c5e14ca6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -77,11 +77,12 @@ export const useChatSend = ({ ...currentConversation, replacements, }); + return replacements; }; const systemPrompt = allSystemPrompts.find((prompt) => prompt.id === editingSystemPromptId); - const userMessages = getCombinedMessage({ + const userMessage = getCombinedMessage({ isNewChat: currentConversation.messages.length === 0, currentReplacements: currentConversation.replacements, promptText, @@ -90,16 +91,16 @@ export const useChatSend = ({ onNewReplacements, }); - const updatedMessages = [...currentConversation.messages, ...userMessages]; + const updatedMessages = [...currentConversation.messages, userMessage].map((m) => ({ + ...m, + content: getMessageContentWithoutReplacements({ + messageContent: m.content ?? '', + replacements, + }), + })); setCurrentConversation({ ...currentConversation, - messages: updatedMessages.map((m) => ({ - ...m, - content: getMessageContentWithoutReplacements({ - messageContent: m.content ?? '', - replacements, - }), - })), + messages: updatedMessages, }); // Reset prompt context selection and preview before sending: @@ -109,7 +110,7 @@ export const useChatSend = ({ const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, - messages: userMessages, + messages: [userMessage], conversationId: currentConversation.id, replacements, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index 49ff9398708b3..d426433e54b01 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -5,11 +5,7 @@ * 2.0. */ -import { - getAnonymizedData, - getAnonymizedValues, - getCsvFromData, -} from '@kbn/elastic-assistant-common'; +import { transformRawData } from '@kbn/elastic-assistant-common'; import { getAnonymizedValue as defaultGetAnonymizedValue } from '../get_anonymized_value'; import type { Message } from '../../assistant_context/types'; import type { SelectedPromptContext } from '../prompt_context/types'; @@ -54,40 +50,34 @@ export function getCombinedMessage({ rawValue: string; }) => string; isNewChat: boolean; - onNewReplacements: (newReplacements: Record) => void; + onNewReplacements: ( + newReplacements: Record + ) => Record | undefined; promptText: string; selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; -}): Message[] { - return Object.keys(selectedPromptContexts) +}): Message { + const promptContextsContent = Object.keys(selectedPromptContexts) .sort() .map((id) => { - let content: string; - if (typeof selectedPromptContexts[id].rawData === 'string') { - content = `${SYSTEM_PROMPT_CONTEXT_NON_I18N( - selectedPromptContexts[id].rawData.toString() - )}`; - } else { - const anonymizedData = getAnonymizedData({ - allow: selectedPromptContexts[id].allow, - allowReplacement: selectedPromptContexts[id].allowReplacement, - currentReplacements, - rawData: selectedPromptContexts[id].rawData as Record, - getAnonymizedValue, - getAnonymizedValues, - }); + const promptContext = transformRawData({ + allow: selectedPromptContexts[id].allow, + allowReplacement: selectedPromptContexts[id].allowReplacement, + currentReplacements, + getAnonymizedValue, + onNewReplacements, + rawData: selectedPromptContexts[id].rawData, + }); - const promptContext = getCsvFromData(anonymizedData.anonymizedData); - content = `${ - isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '' - }${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)} - ${promptText}`; - onNewReplacements(anonymizedData.replacements); - } - return { - content, - role: 'user', // we are combining the system and user messages into one message - timestamp: new Date().toLocaleString(), - }; + return `${SYSTEM_PROMPT_CONTEXT_NON_I18N(promptContext)}`; }); + + return { + content: `${ + isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '' + }${promptContextsContent} + ${promptText}`, + role: 'user', // we are combining the system and user messages into one message + timestamp: new Date().toLocaleString(), + }; } From ddd563f3d1eb7e2d7d6a850c6522e7b9cd0ec4c6 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 13:43:16 -0700 Subject: [PATCH 072/141] fix streaming+ tests --- ...ost_actions_connector_execute_route.gen.ts | 3 + .../impl/assistant/api/index.test.tsx | 10 +- .../impl/assistant/api/index.tsx | 37 ++--- .../chat_send/use_chat_send.test.tsx | 20 ++- .../assistant/chat_send/use_chat_send.tsx | 12 +- .../impl/assistant/helpers.test.ts | 3 - .../impl/assistant/helpers.ts | 6 + .../impl/assistant/index.tsx | 7 +- .../use_settings_updater.test.tsx | 2 +- .../assistant/use_conversation/index.test.tsx | 48 +----- .../impl/assistant/use_conversation/index.tsx | 27 --- .../impl/assistant_context/index.tsx | 20 +-- .../server/lib/executor.test.ts | 68 +++++--- .../elastic_assistant/server/lib/executor.ts | 11 +- .../server/lib/parse_stream.test.ts | 102 ++++++++++++ .../server/lib/parse_stream.ts | 157 ++++++++++++++++++ .../post_actions_connector_execute.test.ts | 21 ++- .../assistant/get_comments/index.test.tsx | 2 +- .../public/assistant/get_comments/index.tsx | 31 +--- .../get_comments/stream/index.test.tsx | 4 +- .../assistant/get_comments/stream/index.tsx | 28 ++-- .../get_comments/stream/use_stream.test.tsx | 4 +- .../get_comments/stream/use_stream.tsx | 12 +- 23 files changed, 419 insertions(+), 216 deletions(-) create mode 100644 x-pack/plugins/elastic_assistant/server/lib/parse_stream.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 8a052f0fdd770..218f5ac1ef131 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,6 +64,9 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), + llmType: z.string().refine((value) => value === 'bedrock' || value === 'openai', { + message: "llmType must be either 'bedrock' or 'openai'", + }), }); export type ExecuteConnectorRequestBodyInput = z.input; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx index d1efa0b032d6a..ce07bc3373998 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.test.tsx @@ -54,7 +54,7 @@ describe('API tests', () => { expect(mockHttp.fetch).toHaveBeenCalledWith( '/internal/elastic_assistant/actions/connector/foo/_execute', { - body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"isEnabledKnowledgeBase":true,"isEnabledRAGAlerts":false}', + body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"conversationId":"test","isEnabledKnowledgeBase":true,"isEnabledRAGAlerts":false,"llmType":"openai"}', headers: { 'Content-Type': 'application/json' }, method: 'POST', signal: undefined, @@ -74,7 +74,7 @@ describe('API tests', () => { expect(mockHttp.fetch).toHaveBeenCalledWith( '/internal/elastic_assistant/actions/connector/foo/_execute', { - body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeStream"},"isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":false}', + body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeStream"},"conversationId":"test","isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":false,"llmType":"openai"}', method: 'POST', asResponse: true, rawResponse: true, @@ -100,7 +100,7 @@ describe('API tests', () => { expect(mockHttp.fetch).toHaveBeenCalledWith( '/internal/elastic_assistant/actions/connector/foo/_execute', { - body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"isEnabledKnowledgeBase":true,"isEnabledRAGAlerts":true,"alertsIndexPattern":".alerts-security.alerts-default","allow":["a","b","c"],"allowReplacement":["b","c"],"replacements":{"auuid":"real.hostname"},"size":30}', + body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"conversationId":"test","replacements":{"auuid":"real.hostname"},"isEnabledKnowledgeBase":true,"isEnabledRAGAlerts":true,"llmType":"openai","alertsIndexPattern":".alerts-security.alerts-default","allow":["a","b","c"],"allowReplacement":["b","c"],"size":30}', headers: { 'Content-Type': 'application/json', }, @@ -123,7 +123,7 @@ describe('API tests', () => { expect(mockHttp.fetch).toHaveBeenCalledWith( '/internal/elastic_assistant/actions/connector/foo/_execute', { - body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":false}', + body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"conversationId":"test","isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":false,"llmType":"openai"}', method: 'POST', headers: { 'Content-Type': 'application/json', @@ -146,7 +146,7 @@ describe('API tests', () => { expect(mockHttp.fetch).toHaveBeenCalledWith( '/internal/elastic_assistant/actions/connector/foo/_execute', { - body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":true}', + body: '{"params":{"subActionParams":{"model":"gpt-4","messages":[{"role":"user","content":"This is a test"}],"n":1,"stop":null,"temperature":0.2},"subAction":"invokeAI"},"conversationId":"test","isEnabledKnowledgeBase":false,"isEnabledRAGAlerts":true,"llmType":"openai"}', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx index 939e9f0933bb8..0ce92c5220ffe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/index.tsx @@ -15,6 +15,7 @@ import { getFormattedMessageContent, getOptionalRequestParams, hasParsableResponse, + llmTypeDictionary, } from '../helpers'; export * from './conversations'; @@ -78,6 +79,7 @@ export const fetchConnectorExecuteAction = async ({ messages: outboundMessages, }; + const llmType = llmTypeDictionary[apiConfig.connectorTypeTitle ?? 'OpenAI']; // TODO: Remove in part 3 of streaming work for security solution // tracked here: https://github.com/elastic/security-team/issues/7363 // In part 3 I will make enhancements to langchain to introduce streaming @@ -91,29 +93,18 @@ export const fetchConnectorExecuteAction = async ({ size, }); - const requestBody = isStream - ? { - params: { - subActionParams: body, - subAction: 'invokeStream', - }, - conversationId, - replacements, - isEnabledKnowledgeBase, - isEnabledRAGAlerts, - ...optionalRequestParams, - } - : { - params: { - subActionParams: body, - subAction: 'invokeAI', - }, - conversationId, - replacements, - isEnabledKnowledgeBase, - isEnabledRAGAlerts, - ...optionalRequestParams, - }; + const requestBody = { + params: { + subActionParams: body, + subAction: isStream ? 'invokeStream' : 'invokeAI', + }, + conversationId, + replacements, + isEnabledKnowledgeBase, + isEnabledRAGAlerts, + llmType, + ...optionalRequestParams, + }; try { if (isStream) { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index ebb45a30f41cd..2e87f96d837d4 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -95,14 +95,14 @@ describe('use chat send', () => { await waitFor(() => { expect(sendMessages).toHaveBeenCalled(); - const appendMessageSend = appendMessage.mock.calls[0][0]; - const appendMessageResponse = appendMessage.mock.calls[1][0]; - expect(appendMessageSend.message.content).toEqual( - `You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL or EQL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\nUse the following context to answer questions:\n\n\n\n${promptText}` + const appendMessageSend = setCurrentConversation.mock.calls[1][0].messages[0]; + const appendMessageResponse = setCurrentConversation.mock.calls[1][0].messages[1]; + expect(appendMessageSend.content).toEqual( + `You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL or EQL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\nUse the following context to answer questions:\n\n\n ${promptText}` ); - expect(appendMessageSend.message.role).toEqual('user'); - expect(appendMessageResponse.message.content).toEqual(robotMessage.response); - expect(appendMessageResponse.message.role).toEqual('assistant'); + expect(appendMessageSend.role).toEqual('user'); + expect(appendMessageResponse.content).toEqual(robotMessage.response); + expect(appendMessageResponse.role).toEqual('assistant'); }); }); it('handleButtonSendMessage sends message with only provided prompt text and context already exists in convo history', async () => { @@ -119,7 +119,8 @@ describe('use chat send', () => { await waitFor(() => { expect(sendMessages).toHaveBeenCalled(); - expect(appendMessage.mock.calls[0][0].message.content).toEqual(`\n\n${promptText}`); + const messages = setCurrentConversation.mock.calls[0][0].messages; + expect(messages[messages.length - 1].content).toEqual(`\n ${promptText}`); }); }); it('handleRegenerateResponse removes the last message of the conversation, resends the convo to GenAI, and appends the message received', async () => { @@ -135,7 +136,8 @@ describe('use chat send', () => { await waitFor(() => { expect(sendMessages).toHaveBeenCalled(); - expect(appendMessage.mock.calls[0][0].message.content).toEqual(robotMessage.response); + const messages = setCurrentConversation.mock.calls[1][0].messages; + expect(messages[messages.length - 1].content).toEqual(robotMessage.response); }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index f53b3c5e14ca6..617ae01c91e8b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -136,18 +136,26 @@ export const useChatSend = ({ ); const handleRegenerateResponse = useCallback(async () => { - await removeLastMessage(currentConversation.id); + // remove last message from the local state immediately + setCurrentConversation({ + ...currentConversation, + messages: currentConversation.messages.slice(0, -1), + }); + + const updatedMessages = (await removeLastMessage(currentConversation.id)) ?? []; const rawResponse = await sendMessages({ apiConfig: currentConversation.apiConfig, http, + // do not send any new messages, the previous conversation is already stored messages: [], conversationId: currentConversation.id, }); + const responseMessage: Message = getMessageFromRawResponse(rawResponse); setCurrentConversation({ ...currentConversation, - messages: [...currentConversation.messages, responseMessage], + messages: [...updatedMessages, responseMessage], }); }, [currentConversation, http, removeLastMessage, sendMessages, setCurrentConversation]); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts index 331361679ba75..bed90bdf6c215 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts @@ -242,7 +242,6 @@ describe('getBlockBotConversation', () => { alertsIndexPattern: 'indexPattern', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - replacements: { key: 'value' }, size: 10, }; @@ -257,7 +256,6 @@ describe('getBlockBotConversation', () => { alertsIndexPattern: 'indexPattern', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - replacements: { key: 'value' }, size: 10, }; @@ -267,7 +265,6 @@ describe('getBlockBotConversation', () => { alertsIndexPattern: 'indexPattern', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - replacements: { key: 'value' }, size: 10, }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index b00dd3010418f..bff5fbec68252 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -149,3 +149,9 @@ export const hasParsableResponse = ({ isEnabledRAGAlerts: boolean; isEnabledKnowledgeBase: boolean; }): boolean => isEnabledKnowledgeBase || isEnabledRAGAlerts; + +export const llmTypeDictionary: Record = { + 'Amazon Bedrock': 'bedrock', + 'Azure OpenAI': 'openai', + OpenAI: 'openai', +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 5c7233360a964..fb008ed195655 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -98,8 +98,7 @@ const AssistantComponent: React.FC = ({ baseConversations, } = useAssistantContext(); - const { amendMessage, getDefaultConversation, getConversation, deleteConversation } = - useConversation(); + const { getDefaultConversation, getConversation, deleteConversation } = useConversation(); const [selectedPromptContexts, setSelectedPromptContexts] = useState< Record @@ -479,7 +478,7 @@ const AssistantComponent: React.FC = ({ comments={getComments({ currentConversation, showAnonymizedValues, - amendMessage, + refetchCurrentConversation, regenerateMessage: handleRegenerateResponse, isFetchingResponse: isLoadingChatSend, })} @@ -509,7 +508,7 @@ const AssistantComponent: React.FC = ({ ), [ - amendMessage, + refetchCurrentConversation, currentConversation, editingSystemPromptId, getComments, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx index bdd04fee89ae5..e304c803ac4ef 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/use_settings_updater/use_settings_updater.test.tsx @@ -149,7 +149,7 @@ describe('useSettingsUpdater', () => { await result.current.saveSettings(); expect(mockHttp.fetch).toHaveBeenCalledWith( - '/api/elastic_assistant/conversations/_bulk_action', + '/api/elastic_assistant/current_user/conversations/_bulk_action', { method: 'POST', version: '2023-10-31', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index 91648c721bf08..697f841acaf0f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -52,24 +52,6 @@ describe('useConversation', () => { jest.clearAllMocks(); }); - it('should append a message to an existing conversation when called with valid conversationId and message', async () => { - await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => {children}, - }); - await waitForNextUpdate(); - appendConversationMessagesApi.mockResolvedValue({ - messages: [message, anotherMessage, message], - }); - - const appendResult = await result.current.appendMessage({ - conversationId: welcomeConvo.id, - message, - }); - expect(appendResult).toHaveLength(3); - expect(appendResult![2]).toEqual(message); - }); - }); it('should report telemetry when a message has been sent', async () => { await act(async () => { @@ -94,7 +76,8 @@ describe('useConversation', () => { appendConversationMessagesApi.mockResolvedValue([message, anotherMessage, message]); await result.current.appendMessage({ - conversationId: welcomeConvo.id, + id: 'longuuid', + title: welcomeConvo.id, message, }); expect(reportAssistantMessageSent).toHaveBeenCalledWith({ @@ -214,31 +197,4 @@ describe('useConversation', () => { expect(removeResult).toEqual([message]); }); }); - - it('amendMessage updates the last message of conversation[] for a given conversationId with provided content', async () => { - await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - {children} - ), - }); - await waitForNextUpdate(); - - await result.current.amendMessage({ - conversationId: 'new-convo', - content: 'hello world', - }); - - expect(appendConversationMessagesApi).toHaveBeenCalledWith({ - conversationId: mockConvo.id, - http: httpMock, - messages: [ - { - ...anotherMessage, - content: 'hello world', - }, - ], - }); - }); - }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 09b12efc6d025..81798ad832714 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -32,10 +32,6 @@ interface AppendMessageProps { title: string; message: Message; } -interface AmendMessageProps { - conversationId: string; - content: string; -} interface AppendReplacementsProps { conversationId: string; @@ -54,7 +50,6 @@ interface SetApiConfigProps { interface UseConversation { appendMessage: ({ id, title, message }: AppendMessageProps) => Promise; - amendMessage: ({ conversationId, content }: AmendMessageProps) => Promise; appendReplacements: ({ conversationId, replacements, @@ -106,27 +101,6 @@ export const useConversation = (): UseConversation => { [http] ); - /** - * Updates the last message of conversation[] for a given conversationId with provided content - */ - const amendMessage = useCallback( - async ({ conversationId, content }: AmendMessageProps) => { - const prevConversation = await getConversationById({ http, id: conversationId }); - if (prevConversation != null) { - const { messages } = prevConversation; - const message = messages[messages.length - 1]; - const updatedMessages = message ? [{ ...message, content }] : []; - - await appendConversationMessages({ - http, - conversationId, - messages: updatedMessages, - }); - } - }, - [http] - ); - /** * Append a message to the conversation[] for a given conversationId */ @@ -275,7 +249,6 @@ export const useConversation = (): UseConversation => { ); return { - amendMessage, appendMessage, appendReplacements, clearConversation, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index 5c888abe50537..3129bf195c051 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -68,21 +68,15 @@ export interface AssistantProviderProps { docLinks: Omit; children: React.ReactNode; getComments: ({ - amendMessage, currentConversation, isFetchingResponse, + refetchCurrentConversation, regenerateMessage, showAnonymizedValues, }: { - amendMessage: ({ - conversationId, - content, - }: { - conversationId: string; - content: string; - }) => Promise; currentConversation: Conversation; isFetchingResponse: boolean; + refetchCurrentConversation: () => void; regenerateMessage: (conversationId: string) => void; showAnonymizedValues: boolean; }) => EuiCommentProps[]; @@ -117,18 +111,12 @@ export interface UseAssistantContext { getComments: ({ currentConversation, showAnonymizedValues, - amendMessage, + refetchCurrentConversation, isFetchingResponse, }: { currentConversation: Conversation; isFetchingResponse: boolean; - amendMessage: ({ - conversationId, - content, - }: { - conversationId: string; - content: string; - }) => Promise; + refetchCurrentConversation: () => void; regenerateMessage: () => void; showAnonymizedValues: boolean; }) => EuiCommentProps[]; diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts index 83b0578d8a77b..84596b92f19f6 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts @@ -21,6 +21,17 @@ const request = { params: {}, }, } as KibanaRequest; +const onMessageSent = jest.fn(); +const connectorId = 'testConnectorId'; +const testProps: Omit = { + params: { + subAction: 'invokeAI', + subActionParams: { messages: [{ content: 'hello', role: 'user' }] }, + }, + request, + connectorId, + onMessageSent, +}; describe('executeAction', () => { beforeEach(() => { @@ -35,16 +46,16 @@ describe('executeAction', () => { }, }), }), - }; - const connectorId = 'testConnectorId'; + } as unknown as Props['actions']; - const result = await executeAction({ actions, request, connectorId } as unknown as Props); + const result = await executeAction({ ...testProps, actions }); expect(result).toEqual({ connector_id: connectorId, data: 'Test message', status: 'ok', }); + expect(onMessageSent).toHaveBeenCalledWith('Test message'); }); it('should execute an action and return a Readable object when the response from the actions framework is a stream', async () => { @@ -55,10 +66,9 @@ describe('executeAction', () => { data: readableStream, }), }), - }; - const connectorId = 'testConnectorId'; + } as unknown as Props['actions']; - const result = await executeAction({ actions, request, connectorId } as unknown as Props); + const result = await executeAction({ ...testProps, actions }); expect(JSON.stringify(result)).toStrictEqual( JSON.stringify(readableStream.pipe(new PassThrough())) @@ -70,12 +80,11 @@ describe('executeAction', () => { getActionsClientWithRequest: jest .fn() .mockRejectedValue(new Error('Failed to retrieve actions client')), - }; - const connectorId = 'testConnectorId'; + } as unknown as Props['actions']; - await expect( - executeAction({ actions, request, connectorId } as unknown as Props) - ).rejects.toThrowError('Failed to retrieve actions client'); + await expect(executeAction({ ...testProps, actions })).rejects.toThrowError( + 'Failed to retrieve actions client' + ); }); it('should throw an error if the actions client fails to execute the action', async () => { @@ -83,12 +92,11 @@ describe('executeAction', () => { getActionsClientWithRequest: jest.fn().mockResolvedValue({ execute: jest.fn().mockRejectedValue(new Error('Failed to execute action')), }), - }; - const connectorId = 'testConnectorId'; + } as unknown as Props['actions']; - await expect( - executeAction({ actions, request, connectorId } as unknown as Props) - ).rejects.toThrowError('Failed to execute action'); + await expect(executeAction({ ...testProps, actions })).rejects.toThrowError( + 'Failed to execute action' + ); }); it('should throw an error when the response from the actions framework is null or undefined', async () => { @@ -98,11 +106,10 @@ describe('executeAction', () => { data: null, }), }), - }; - const connectorId = 'testConnectorId'; + } as unknown as Props['actions']; try { - await executeAction({ actions, request, connectorId } as unknown as Props); + await executeAction({ ...testProps, actions }); } catch (e) { expect(e.message).toBe('Action result status is error: result is not streamable'); } @@ -118,11 +125,14 @@ describe('executeAction', () => { }), }), } as unknown as ActionsPluginStart; - const connectorId = '12345'; - await expect(executeAction({ actions, request, connectorId })).rejects.toThrowError( - 'Action result status is error: Error message - Service error message' - ); + await expect( + executeAction({ + ...testProps, + actions, + connectorId: '12345', + }) + ).rejects.toThrowError('Action result status is error: Error message - Service error message'); }); it('should throw an error if content of response data is not a string or streamable', async () => { @@ -136,10 +146,14 @@ describe('executeAction', () => { }), }), } as unknown as ActionsPluginStart; - const connectorId = '12345'; - await expect(executeAction({ actions, request, connectorId })).rejects.toThrowError( - 'Action result status is error: result is not streamable' - ); + await expect( + executeAction({ + ...testProps, + + actions, + connectorId: '12345', + }) + ).rejects.toThrowError('Action result status is error: result is not streamable'); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index 9996599f9266f..ed2ce63ce6639 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -13,6 +13,7 @@ import { ConnectorExecutionParams, ExecuteConnectorRequestBody, } from '@kbn/elastic-assistant-common'; +import { handleStreamStorage } from './parse_stream'; export interface Props { onMessageSent: (content: string) => void; @@ -55,16 +56,14 @@ export const executeAction = async ({ status: 'ok', }; } - const readable = get('data', actionResult) as Readable; - readable.read().then(({ done, value }) => { - if (done) { - onMessageSent(value); - } - }); + const readable = get('data', actionResult) as Readable; if (typeof readable?.read !== 'function') { throw new Error('Action result status is error: result is not streamable'); } + // do not await, blocks stream for UI + handleStreamStorage(readable, request.body.llmType, onMessageSent); + return readable.pipe(new PassThrough()); }; diff --git a/x-pack/plugins/elastic_assistant/server/lib/parse_stream.test.ts b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.test.ts new file mode 100644 index 0000000000000..079824b11416d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Transform } from 'stream'; +import { handleStreamStorage } from './parse_stream'; +import { EventStreamCodec } from '@smithy/eventstream-codec'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; + +function createStreamMock() { + const transform: Transform = new Transform({}); + + return { + write: (data: unknown) => { + transform.push(data); + }, + fail: () => { + transform.emit('error', new Error('Stream failed')); + transform.end(); + }, + transform, + complete: () => { + transform.end(); + }, + }; +} +const onMessageSent = jest.fn(); +describe('handleStreamStorage', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + let stream: ReturnType; + + const chunk = { + object: 'chat.completion.chunk', + choices: [ + { + delta: { + content: 'Single.', + }, + }, + ], + }; + + describe('OpenAI stream', () => { + beforeEach(() => { + stream = createStreamMock(); + stream.write(`data: ${JSON.stringify(chunk)}`); + }); + + it('saves the final string successful streaming event', async () => { + stream.complete(); + await handleStreamStorage(stream.transform, 'openai', onMessageSent); + expect(onMessageSent).toHaveBeenCalledWith('Single.'); + }); + it('saves the error message on a failed streaming event', async () => { + const tokenPromise = handleStreamStorage(stream.transform, 'openai', onMessageSent); + + stream.fail(); + await expect(tokenPromise).resolves.not.toThrow(); + expect(onMessageSent).toHaveBeenCalledWith( + `An error occurred while streaming the response:\n\nStream failed` + ); + }); + }); + describe('Bedrock stream', () => { + beforeEach(() => { + stream = createStreamMock(); + stream.write(encodeBedrockResponse('Simple.')); + }); + + it('saves the final string successful streaming event', async () => { + stream.complete(); + await handleStreamStorage(stream.transform, 'bedrock', onMessageSent); + expect(onMessageSent).toHaveBeenCalledWith('Simple.'); + }); + it('saves the error message on a failed streaming event', async () => { + const tokenPromise = handleStreamStorage(stream.transform, 'bedrock', onMessageSent); + + stream.fail(); + await expect(tokenPromise).resolves.not.toThrow(); + expect(onMessageSent).toHaveBeenCalledWith( + `An error occurred while streaming the response:\n\nStream failed` + ); + }); + }); +}); + +function encodeBedrockResponse(completion: string) { + return new EventStreamCodec(toUtf8, fromUtf8).encode({ + headers: {}, + body: Uint8Array.from( + Buffer.from( + JSON.stringify({ + bytes: Buffer.from(JSON.stringify({ completion })).toString('base64'), + }) + ) + ), + }); +} diff --git a/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts new file mode 100644 index 0000000000000..58123e3db8ce0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import { EventStreamCodec } from '@smithy/eventstream-codec'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import { finished } from 'stream/promises'; + +type StreamParser = (responseStream: Readable) => Promise; + +export const handleStreamStorage: ( + responseStream: Readable, + llmType: string, + onMessageSent: (content: string) => void +) => Promise = async (responseStream, llmType, onMessageSent) => { + try { + const parser = llmType === 'bedrock' ? parseBedrockStream : parseOpenAIStream; + // TODO @steph add abort signal + const parsedResponse = await parser(responseStream); + onMessageSent(parsedResponse); + } catch (e) { + onMessageSent(`An error occurred while streaming the response:\n\n${e.message}`); + } +}; + +const parseOpenAIStream: StreamParser = async (stream) => { + let responseBody = ''; + stream.on('data', (chunk) => { + responseBody += chunk.toString(); + }); + return new Promise((resolve, reject) => { + stream.on('end', () => { + resolve(parseOpenAIResponse(responseBody)); + }); + stream.on('error', (err) => { + reject(err); + }); + }); +}; + +const parseOpenAIResponse = (responseBody: string) => + responseBody + .split('\n') + .filter((line) => { + return line.startsWith('data: ') && !line.endsWith('[DONE]'); + }) + .map((line) => { + return JSON.parse(line.replace('data: ', '')); + }) + .filter( + ( + line + ): line is { + choices: Array<{ + delta: { content?: string; function_call?: { name?: string; arguments: string } }; + }>; + } => { + return 'object' in line && line.object === 'chat.completion.chunk'; + } + ) + .reduce((prev, line) => { + const msg = line.choices[0].delta; + return prev + (msg.content || ''); + }, ''); + +const parseBedrockStream: StreamParser = async (responseStream) => { + const responseBuffer: Uint8Array[] = []; + responseStream.on('data', (chunk) => { + // special encoding for bedrock, do not attempt to convert to string + responseBuffer.push(chunk); + }); + await finished(responseStream); + + return parseBedrockBuffer(responseBuffer); +}; + +/** + * Parses a Bedrock buffer from an array of chunks. + * + * @param {Uint8Array[]} chunks - Array of Uint8Array chunks to be parsed. + * @returns {string} - Parsed string from the Bedrock buffer. + */ +const parseBedrockBuffer = (chunks: Uint8Array[]): string => { + // Initialize an empty Uint8Array to store the concatenated buffer. + let bedrockBuffer: Uint8Array = new Uint8Array(0); + + // Map through each chunk to process the Bedrock buffer. + return chunks + .map((chunk) => { + // Concatenate the current chunk to the existing buffer. + bedrockBuffer = concatChunks(bedrockBuffer, chunk); + // Get the length of the next message in the buffer. + let messageLength = getMessageLength(bedrockBuffer); + // Initialize an array to store fully formed message chunks. + const buildChunks = []; + // Process the buffer until no complete messages are left. + while (bedrockBuffer.byteLength > 0 && bedrockBuffer.byteLength >= messageLength) { + // Extract a chunk of the specified length from the buffer. + const extractedChunk = bedrockBuffer.slice(0, messageLength); + // Add the extracted chunk to the array of fully formed message chunks. + buildChunks.push(extractedChunk); + // Remove the processed chunk from the buffer. + bedrockBuffer = bedrockBuffer.slice(messageLength); + // Get the length of the next message in the updated buffer. + messageLength = getMessageLength(bedrockBuffer); + } + + const awsDecoder = new EventStreamCodec(toUtf8, fromUtf8); + + // Decode and parse each message chunk, extracting the 'completion' property. + return buildChunks + .map((bChunk) => { + const event = awsDecoder.decode(bChunk); + const body = JSON.parse( + Buffer.from(JSON.parse(new TextDecoder().decode(event.body)).bytes, 'base64').toString() + ); + return body.completion; + }) + .join(''); + }) + .join(''); +}; + +/** + * Concatenates two Uint8Array buffers. + * + * @param {Uint8Array} a - First buffer. + * @param {Uint8Array} b - Second buffer. + * @returns {Uint8Array} - Concatenated buffer. + */ +function concatChunks(a: Uint8Array, b: Uint8Array): Uint8Array { + const newBuffer = new Uint8Array(a.length + b.length); + // Copy the contents of the first buffer to the new buffer. + newBuffer.set(a); + // Copy the contents of the second buffer to the new buffer starting from the end of the first buffer. + newBuffer.set(b, a.length); + return newBuffer; +} + +/** + * Gets the length of the next message from the buffer. + * + * @param {Uint8Array} buffer - Buffer containing the message. + * @returns {number} - Length of the next message. + */ +function getMessageLength(buffer: Uint8Array): number { + // If the buffer is empty, return 0. + if (buffer.byteLength === 0) return 0; + // Create a DataView to read the Uint32 value at the beginning of the buffer. + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + // Read and return the Uint32 value (message length). + return view.getUint32(0, false); +} diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts index 717dcf131e823..b53814d3b5b30 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts @@ -19,6 +19,7 @@ import { INVOKE_ASSISTANT_ERROR_EVENT, INVOKE_ASSISTANT_SUCCESS_EVENT, } from '../lib/telemetry/event_based_telemetry'; +import { getConversationResponseMock } from '../conversations_data_client/update_conversation.test'; jest.mock('../lib/build_response', () => ({ buildResponse: jest.fn().mockImplementation((x) => x), @@ -62,7 +63,7 @@ jest.mock('../lib/langchain/execute_custom_llm_chain', () => ({ } ), })); - +const existingConversation = getConversationResponseMock(); const reportEvent = jest.fn(); const mockContext = { elasticAssistant: { @@ -70,6 +71,24 @@ const mockContext = { getRegisteredTools: jest.fn(() => []), logger: loggingSystemMock.createLogger(), telemetry: { ...coreMock.createSetup().analytics, reportEvent }, + getCurrentUser: () => ({ + username: 'user', + email: 'email', + fullName: 'full name', + roles: ['user-role'], + enabled: true, + authentication_realm: { name: 'native1', type: 'native' }, + lookup_realm: { name: 'native1', type: 'native' }, + authentication_provider: { type: 'basic', name: 'basic1' }, + authentication_type: 'realm', + elastic_cloud_user: false, + metadata: { _reserved: false }, + }), + getAIAssistantConversationsDataClient: jest.fn().mockResolvedValue({ + getConversation: jest.fn().mockResolvedValue(existingConversation), + updateConversation: jest.fn().mockResolvedValue(existingConversation), + appendConversationMessages: jest.fn().mockResolvedValue(existingConversation), + }), }, core: { elasticsearch: { diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx index 63bd155704553..12e95b321594b 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.test.tsx @@ -24,7 +24,7 @@ const currentConversation = { }; const showAnonymizedValues = false; const testProps = { - amendMessage: jest.fn(), + refetchCurrentConversation: jest.fn(), regenerateMessage: jest.fn(), isFetchingResponse: false, currentConversation, diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index 4dbada304d586..a9d4f648bb2d5 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -43,31 +43,18 @@ const transformMessageWithReplacements = ({ }; export const getComments = ({ - amendMessage, currentConversation, isFetchingResponse, + refetchCurrentConversation, regenerateMessage, showAnonymizedValues, }: { - amendMessage: ({ - conversationId, - content, - }: { - conversationId: string; - content: string; - }) => Promise; currentConversation: Conversation; isFetchingResponse: boolean; + refetchCurrentConversation: () => void; regenerateMessage: (conversationId: string) => void; showAnonymizedValues: boolean; }): EuiCommentProps[] => { - const amendMessageOfConversation = async (content: string) => { - await amendMessage({ - conversationId: currentConversation.id, - content, - }); - }; - const regenerateMessageOfConversation = () => { regenerateMessage(currentConversation.id); }; @@ -82,11 +69,10 @@ export const getComments = ({ timestamp: '...', children: ( ({ content: '' } as unknown as ContentMessage)} isFetching // we never need to append to a code block in the loading comment, which is what this index is used for @@ -116,6 +102,8 @@ export const getComments = ({ eventColor: message.isError ? 'danger' : undefined, }; + const isControlsEnabled = isLastComment && !isUser; + const transformMessage = (content: string) => transformMessageWithReplacements({ message, @@ -130,12 +118,12 @@ export const getComments = ({ ...messageProps, children: ( @@ -151,14 +139,13 @@ export const getComments = ({ actions: , children: ( ), diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx index 29570959fb839..8e93fd69d1fd3 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx @@ -15,10 +15,10 @@ jest.mock('./use_stream'); const content = 'Test Content'; const testProps = { - amendMessage: jest.fn(), + refetchCurrentConversation: jest.fn(), content, index: 1, - isLastComment: true, + isControlsEnabled: true, connectorTypeTitle: 'OpenAI', regenerateMessage: jest.fn(), transformMessage: jest.fn(), diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx index 5cc107068ae19..1ee37aa618827 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx @@ -15,49 +15,49 @@ import { MessagePanel } from './message_panel'; import { MessageText } from './message_text'; interface Props { - amendMessage: (message: string) => Promise; content?: string; isError?: boolean; isFetching?: boolean; - isLastComment: boolean; + isControlsEnabled?: boolean; index: number; connectorTypeTitle: string; reader?: ReadableStreamDefaultReader; + refetchCurrentConversation: () => void; regenerateMessage: () => void; transformMessage: (message: string) => ContentMessage; } export const StreamComment = ({ - amendMessage, content, connectorTypeTitle, index, + isControlsEnabled = false, isError = false, isFetching = false, - isLastComment, reader, + refetchCurrentConversation, regenerateMessage, transformMessage, }: Props) => { const { error, isLoading, isStreaming, pendingMessage, setComplete } = useStream({ - amendMessage, + refetchCurrentConversation, content, connectorTypeTitle, reader, isError, }); - const currentState = useRef({ isStreaming, pendingMessage, amendMessage }); + const currentState = useRef({ isStreaming, pendingMessage, refetchCurrentConversation }); useEffect(() => { - currentState.current = { isStreaming, pendingMessage, amendMessage }; - }, [amendMessage, isStreaming, pendingMessage]); + currentState.current = { isStreaming, pendingMessage, refetchCurrentConversation }; + }, [refetchCurrentConversation, isStreaming, pendingMessage]); useEffect( () => () => { - // if the component is unmounted while streaming, amend the message with the pending message - if (currentState.current.isStreaming && currentState.current.pendingMessage.length > 0) { - currentState.current.amendMessage(currentState.current.pendingMessage ?? ''); + // if the component is unmounted while streaming, fetch the convo to get the completed stream + if (currentState.current.isStreaming) { + currentState.current.refetchCurrentConversation(); } }, // store values in currentState to detect true unmount @@ -74,10 +74,10 @@ export const StreamComment = ({ [isFetching, isLoading, isStreaming] ); const controls = useMemo(() => { - if (reader == null || !isLastComment) { + if (!isControlsEnabled) { return; } - if (isAnythingLoading) { + if (isAnythingLoading && reader) { return ( { @@ -93,7 +93,7 @@ export const StreamComment = ({ ); - }, [isAnythingLoading, isLastComment, reader, regenerateMessage, setComplete]); + }, [isAnythingLoading, isControlsEnabled, reader, regenerateMessage, setComplete]); return ( ; const defaultProps = { - amendMessage, + refetchCurrentConversation, reader: readerComplete, isError: false, connectorTypeTitle: 'OpenAI', diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx index 9271758a8558e..bc75629deaf1a 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx @@ -10,7 +10,7 @@ import type { Subscription } from 'rxjs'; import { getPlaceholderObservable, getStreamObservable } from './stream_observable'; interface UseStreamProps { - amendMessage: (message: string) => void; + refetchCurrentConversation: () => void; isError: boolean; content?: string; connectorTypeTitle: string; @@ -31,16 +31,17 @@ interface UseStream { /** * A hook that takes a ReadableStreamDefaultReader and returns an object with properties and functions * that can be used to handle streaming data from a readable stream - * @param amendMessage - handles the amended message * @param content - the content of the message. If provided, the function will not use the reader to stream data. + * @param connectorTypeTitle - the title of the connector type + * @param refetchCurrentConversation - refetch the current conversation * @param reader - The readable stream reader used to stream data. If provided, the function will use this reader to stream data. * @param isError - indicates whether the reader response is an error message or not */ export const useStream = ({ - amendMessage, content, connectorTypeTitle, reader, + refetchCurrentConversation, isError, }: UseStreamProps): UseStream => { const [pendingMessage, setPendingMessage] = useState(); @@ -57,8 +58,9 @@ export const useStream = ({ const onCompleteStream = useCallback(() => { subscription?.unsubscribe(); setLoading(false); - amendMessage(pendingMessage ?? ''); - }, [amendMessage, pendingMessage, subscription]); + refetchCurrentConversation(); + }, [refetchCurrentConversation, subscription]); + const [complete, setComplete] = useState(false); useEffect(() => { if (complete) { From ef170665cee9d0ed3c2892e70fadff8c7722fe2e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 12 Feb 2024 21:14:46 +0000 Subject: [PATCH 073/141] [CI] Auto-commit changed files from 'yarn openapi:generate' --- .../post_actions_connector_execute_route.gen.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 218f5ac1ef131..8a052f0fdd770 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,9 +64,6 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), - llmType: z.string().refine((value) => value === 'bedrock' || value === 'openai', { - message: "llmType must be either 'bedrock' or 'openai'", - }), }); export type ExecuteConnectorRequestBodyInput = z.input; From 5737ebec87f946c530d1dc9360b55c7cdf888cc0 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 14:21:37 -0700 Subject: [PATCH 074/141] fix spacing --- .../impl/assistant/chat_send/use_chat_send.test.tsx | 4 ++-- .../kbn-elastic-assistant/impl/assistant/prompt/helpers.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index 2e87f96d837d4..b865200017eb8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -98,7 +98,7 @@ describe('use chat send', () => { const appendMessageSend = setCurrentConversation.mock.calls[1][0].messages[0]; const appendMessageResponse = setCurrentConversation.mock.calls[1][0].messages[1]; expect(appendMessageSend.content).toEqual( - `You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL or EQL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\nUse the following context to answer questions:\n\n\n ${promptText}` + `You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL or EQL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\nUse the following context to answer questions:\n\n\n\n${promptText}` ); expect(appendMessageSend.role).toEqual('user'); expect(appendMessageResponse.content).toEqual(robotMessage.response); @@ -120,7 +120,7 @@ describe('use chat send', () => { await waitFor(() => { expect(sendMessages).toHaveBeenCalled(); const messages = setCurrentConversation.mock.calls[0][0].messages; - expect(messages[messages.length - 1].content).toEqual(`\n ${promptText}`); + expect(messages[messages.length - 1].content).toEqual(`\n\n${promptText}`); }); }); it('handleRegenerateResponse removes the last message of the conversation, resends the convo to GenAI, and appends the message received', async () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index d426433e54b01..2774f94bbfbec 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -75,8 +75,7 @@ export function getCombinedMessage({ return { content: `${ isNewChat ? `${selectedSystemPrompt?.content ?? ''}\n\n` : '' - }${promptContextsContent} - ${promptText}`, + }${promptContextsContent}\n\n${promptText}`, role: 'user', // we are combining the system and user messages into one message timestamp: new Date().toLocaleString(), }; From 424e0db2afc52989ee5a86b1a9e657ccd110fc93 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 14:25:07 -0700 Subject: [PATCH 075/141] fix more tests --- .../impl/assistant/api/conversations/conversations.test.tsx | 4 ++-- .../use_fetch_current_user_conversations.test.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx index 4d7c8fe3a0d24..746856ef18db7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.test.tsx @@ -36,7 +36,7 @@ describe('conversations api', () => { await waitForNextUpdate(); expect(deleteProps.http.fetch).toHaveBeenCalledWith( - '/api/elastic_assistant/conversations/test', + '/api/elastic_assistant/current_user/conversations/test', { method: 'DELETE', signal: undefined, @@ -66,7 +66,7 @@ describe('conversations api', () => { await waitForNextUpdate(); expect(getProps.http.fetch).toHaveBeenCalledWith( - '/api/elastic_assistant/conversations/test', + '/api/elastic_assistant/current_user/conversations/test', { method: 'GET', signal: undefined, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx index 06aaa41a4b118..b890fe57247fe 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_fetch_current_user_conversations.test.tsx @@ -44,7 +44,7 @@ describe('useFetchCurrentUserConversations', () => { ); await waitForNextUpdate(); expect(defaultProps.http.fetch).toHaveBeenCalledWith( - '/api/elastic_assistant/conversations/current_user/_find', + '/api/elastic_assistant/current_user/conversations/_find', { method: 'GET', query: { From dcb978630d08fb23055fd1ba14445fdfaa77ff07 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 14:27:12 -0700 Subject: [PATCH 076/141] try fixing zod --- .../post_actions_connector_execute_route.gen.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 8a052f0fdd770..3c8a182584f08 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,6 +64,7 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), + llmType: z.string(), }); export type ExecuteConnectorRequestBodyInput = z.input; From 289d1dc711cf7aa21076871b85c71c5d48baccaf Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 14:38:31 -0700 Subject: [PATCH 077/141] Revert "try fixing zod" This reverts commit dcb978630d08fb23055fd1ba14445fdfaa77ff07. --- .../post_actions_connector_execute_route.gen.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 3c8a182584f08..8a052f0fdd770 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,7 +64,6 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), - llmType: z.string(), }); export type ExecuteConnectorRequestBodyInput = z.input; From 71d41178a5d3f5ed0c8266ba48bf2825575535bb Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 12 Feb 2024 14:41:04 -0700 Subject: [PATCH 078/141] fix zod --- .../post_actions_connector_execute_route.gen.ts | 1 + ...st_actions_connector_execute_route.schema.yaml | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 8a052f0fdd770..cd31b266a6773 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,6 +64,7 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), + llmType: z.enum(['bedrock', 'openai']).optional(), }); export type ExecuteConnectorRequestBodyInput = z.input; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index 059ce13304f75..fac9c5189225f 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -51,6 +51,11 @@ paths: additionalProperties: true size: type: number + llmType: + type: string + enum: + - bedrock + - openai responses: '200': description: Successful response @@ -59,11 +64,11 @@ paths: schema: type: object properties: - data: + data: type: string connector_id: type: string - replacements: + replacements: type: object additionalProperties: true status: @@ -105,7 +110,7 @@ components: - subActionParams - subAction properties: - subActionParams: + subActionParams: type: object required: - messages @@ -122,7 +127,7 @@ components: type: string description: Message role. enum: - - system + - system - user - assistant content: @@ -138,4 +143,4 @@ components: temperature: type: number subAction: - type: string \ No newline at end of file + type: string From b8a0173e072d5bbfa0a629ccd65dab56e4ebba14 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 09:24:25 -0800 Subject: [PATCH 079/141] fixed connectorTitle save on bulk update/create --- .../transform_raw_data/index.test.tsx | 6 ++--- .../transform_raw_data/index.tsx | 2 +- ...ost_actions_connector_execute_route.gen.ts | 2 +- ...ctions_connector_execute_route.schema.yaml | 1 + .../conversations/common_attributes.gen.ts | 20 ---------------- .../common_attributes.schema.yaml | 16 ------------- .../assistant/chat_send/use_chat_send.tsx | 4 ++-- .../conversation_settings.tsx | 1 + .../impl/assistant/prompt/helpers.ts | 4 +--- .../create_conversation.test.ts | 17 +++++++++---- .../create_conversation.ts | 8 ++++--- .../conversations_data_client/transforms.ts | 16 ------------- .../update_conversation.test.ts | 24 +++++++++---------- .../update_conversation.ts | 8 ++++--- 14 files changed, 45 insertions(+), 84 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx index ded518deece66..fb8b1c5b42f6b 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.test.tsx @@ -22,7 +22,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: jest.fn(), + onNewReplacements: () => {}, rawData: inputRawData.rawData, }); @@ -64,7 +64,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: jest.fn(), + onNewReplacements: () => {}, rawData: inputRawData.rawData, }); @@ -88,7 +88,7 @@ describe('transformRawData', () => { allowReplacement: inputRawData.allowReplacement, currentReplacements: {}, getAnonymizedValue: mockGetAnonymizedValue, - onNewReplacements: jest.fn(), + onNewReplacements: () => {}, rawData: inputRawData.rawData, }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx index 5846e054fbab7..f1fe5e9331344 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/data_anonymization/transform_raw_data/index.tsx @@ -27,7 +27,7 @@ export const transformRawData = ({ currentReplacements: Record | undefined; rawValue: string; }) => string; - onNewReplacements: (replacements: Record) => Record | undefined; + onNewReplacements?: (replacements: Record) => void; rawData: string | Record; }): string => { if (typeof rawData === 'string') { diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index cd31b266a6773..97fd0a663a79f 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -64,7 +64,7 @@ export const ExecuteConnectorRequestBody = z.object({ isEnabledRAGAlerts: z.boolean().optional(), replacements: z.object({}).catchall(z.unknown()).optional(), size: z.number().optional(), - llmType: z.enum(['bedrock', 'openai']).optional(), + llmType: z.enum(['bedrock', 'openai']), }); export type ExecuteConnectorRequestBodyInput = z.input; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index fac9c5189225f..f50074e322320 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -27,6 +27,7 @@ paths: required: - params - conversationId + - llmType properties: conversationId: $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/UUID' diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts index 87b656c4e0096..5b3ba908cb3d4 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts @@ -46,21 +46,6 @@ export const User = z.object({ name: z.string().optional(), }); -/** - * Could be any string, not necessarily a UUID - */ -export type MessagePresentation = z.infer; -export const MessagePresentation = z.object({ - /** - * Could be any string, not necessarily a UUID - */ - delay: z.number().int().optional(), - /** - * Could be any string, not necessarily a UUID - */ - stream: z.boolean().optional(), -}); - /** * trace Data */ @@ -111,7 +96,6 @@ export const Message = z.object({ * Message content. */ reader: Reader.optional(), - replacements: Replacement.optional(), /** * Message role. */ @@ -124,10 +108,6 @@ export const Message = z.object({ * Is error message. */ isError: z.boolean().optional(), - /** - * ID of the exception container - */ - presentation: MessagePresentation.optional(), /** * trace Data */ diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml index e2208b8452c5b..1dd58fbf941c6 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml @@ -28,17 +28,6 @@ components: type: string description: User name. - MessagePresentation: - type: object - description: Could be any string, not necessarily a UUID - properties: - delay: - type: integer - description: Could be any string, not necessarily a UUID - stream: - type: boolean - description: Could be any string, not necessarily a UUID - TraceData: type: object description: trace Data @@ -87,8 +76,6 @@ components: reader: $ref: '#/components/schemas/Reader' description: Message content. - replacements: - $ref: '#/components/schemas/Replacement' role: $ref: '#/components/schemas/MessageRole' description: Message role. @@ -98,9 +85,6 @@ components: isError: type: boolean description: Is error message. - presentation: - $ref: '#/components/schemas/MessagePresentation' - description: ID of the exception container traceData: $ref: '#/components/schemas/TraceData' description: trace Data diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 617ae01c91e8b..e8a2b95fb6acb 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -77,7 +77,6 @@ export const useChatSend = ({ ...currentConversation, replacements, }); - return replacements; }; const systemPrompt = allSystemPrompts.find((prompt) => prompt.id === editingSystemPromptId); @@ -176,9 +175,10 @@ export const useChatSend = ({ setPromptTextPreview(''); setUserPrompt(''); setSelectedPromptContexts({}); - setEditingSystemPromptId(defaultSystemPromptId); await clearConversation(currentConversation.id); await refresh(); + + setEditingSystemPromptId(defaultSystemPromptId); }, [ allSystemPrompts, clearConversation, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index 1c30be18e477f..e3713cbb181f1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -221,6 +221,7 @@ export const ConversationSettings: React.FC = React.m apiConfig: { ...selectedConversation.apiConfig, connectorId: connector?.id, + connectorTypeTitle: connector?.connectorTypeTitle, provider: config?.apiProvider, model: config?.defaultModel, }, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts index 2774f94bbfbec..0c9299476c9d8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt/helpers.ts @@ -50,9 +50,7 @@ export function getCombinedMessage({ rawValue: string; }) => string; isNewChat: boolean; - onNewReplacements: ( - newReplacements: Record - ) => Record | undefined; + onNewReplacements: (newReplacements: Record) => void; promptText: string; selectedPromptContexts: Record; selectedSystemPrompt: Prompt | undefined; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index 0cfda1c5fd8c8..9daccb0039c65 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -12,11 +12,20 @@ import { estypes } from '@elastic/elasticsearch'; import { SearchEsConversationSchema } from './types'; import { getConversation } from './get_conversation'; import { ConversationCreateProps, ConversationResponse } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; jest.mock('./get_conversation', () => ({ getConversation: jest.fn(), })); +const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, +} as AuthenticatedUser; + export const getCreateConversationMock = (): ConversationCreateProps => ({ title: 'test', apiConfig: { @@ -141,7 +150,7 @@ describe('createConversation', () => { esClient, conversationIndex: 'index-1', spaceId: 'test', - user: { name: 'test' }, + user: mockUser1, conversation, logger, }); @@ -174,7 +183,7 @@ describe('createConversation', () => { esClient, conversationIndex: 'index-1', spaceId: 'test', - user: { name: 'test' }, + user: mockUser1, conversation, logger, }); @@ -196,7 +205,7 @@ describe('createConversation', () => { esClient, conversationIndex: 'index-1', spaceId: 'test', - user: { name: 'test' }, + user: mockUser1, conversation, logger, }); @@ -220,7 +229,7 @@ describe('createConversation', () => { esClient, conversationIndex: 'index-1', spaceId: 'test', - user: { name: 'test' }, + user: mockUser1, conversation, logger, }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index ae5f4e9b53ee5..21a67b8621f01 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -15,6 +15,7 @@ import { Provider, Reader, Replacement, + getMessageContentWithoutReplacements, } from '@kbn/elastic-assistant-common'; import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; import { getConversation } from './get_conversation'; @@ -131,11 +132,12 @@ export const transformToCreateScheme = ( is_default: isDefault, messages: messages?.map((message) => ({ '@timestamp': message.timestamp, - content: message.content, + content: getMessageContentWithoutReplacements({ + messageContent: message.content, + replacements: replacements as Record | undefined, + }), is_error: message.isError, - presentation: message.presentation, reader: message.reader, - replacements: message.replacements, role: message.role, trace_data: { trace_id: message.traceData?.traceId, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index 8848c2efe9fbc..d668cec60b0e6 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -41,9 +41,7 @@ export const transformESToConversations = ( timestamp: message['@timestamp'], content: message.content, ...(message.is_error ? { isError: message.is_error } : {}), - ...(message.presentation ? { presentation: message.presentation } : {}), ...(message.reader ? { reader: message.reader } : {}), - ...(message.replacements ? { replacements: message.replacements as Replacement } : {}), role: message.role, ...(message.trace_data ? { @@ -63,17 +61,3 @@ export const transformESToConversations = ( return conversation; }); }; - -export const encodeHitVersion = (hit: T): string | undefined => { - // Have to do this "as cast" here as these two types aren't included in the SearchResponse hit type - const { _seq_no: seqNo, _primary_term: primaryTerm } = hit as unknown as { - _seq_no: number; - _primary_term: number; - }; - - if (seqNo == null || primaryTerm == null) { - return undefined; - } else { - return Buffer.from(JSON.stringify([seqNo, primaryTerm]), 'utf8').toString('base64'); - } -}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 50208e2c17b55..0c13163d6ce4a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -10,6 +10,7 @@ import { loggerMock } from '@kbn/logging-mocks'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; import { ConversationResponse, ConversationUpdateProps } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; export const getUpdateConversationOptionsMock = (): ConversationUpdateProps => ({ id: 'test', @@ -27,6 +28,14 @@ export const getUpdateConversationOptionsMock = (): ConversationUpdateProps => ( replacements: {} as any, }); +const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, +} as AuthenticatedUser; + export const getConversationResponseMock = (): ConversationResponse => ({ id: 'test', title: 'test', @@ -79,10 +88,7 @@ describe('updateConversation', () => { conversationIndex: 'index-1', existingConversation, conversationUpdateProps: conversation, - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + user: mockUser1, }); const expected: ConversationResponse = { ...getConversationResponseMock(), @@ -104,10 +110,7 @@ describe('updateConversation', () => { conversationIndex: 'index-1', existingConversation, conversationUpdateProps: conversation, - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + user: mockUser1, }); expect(updatedList).toEqual(null); }); @@ -125,10 +128,7 @@ describe('updateConversation', () => { conversationIndex: 'index-1', existingConversation, conversationUpdateProps: conversation, - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + user: mockUser1, }) ).rejects.toThrow('No conversation has been updated'); }); diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index 47313d18e53a2..bca251fa098b6 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -13,6 +13,7 @@ import { ConversationUpdateProps, Provider, MessageRole, + getMessageContentWithoutReplacements, } from '@kbn/elastic-assistant-common'; import { AuthenticatedUser } from '@kbn/security-plugin/common'; import { getConversation } from './get_conversation'; @@ -186,11 +187,12 @@ export const transformToUpdateScheme = ( replacements, messages: messages?.map((message) => ({ '@timestamp': message.timestamp, - content: message.content, + content: getMessageContentWithoutReplacements({ + messageContent: message.content, + replacements: replacements as Record | undefined, + }), is_error: message.isError, - presentation: message.presentation, reader: message.reader, - replacements: message.replacements, role: message.role, trace_data: { trace_id: message.traceData?.traceId, From 52772b6b8e5103a0d12fa353cf0350b45ac7d1a8 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 14:42:19 -0800 Subject: [PATCH 080/141] used datastream for prompts and anonymization fields --- .../current_fields.json | 23 - .../current_mappings.json | 4422 ++++++++--------- .../check_registered_types.test.ts | 2 - .../group3/type_registrations.test.ts | 2 - .../group5/dot_kibana_split.test.ts | 2 - .../kbn-elastic-assistant-common/constants.ts | 2 +- ...ulk_crud_anonymization_fields_route.gen.ts | 14 +- ...rud_anonymization_fields_route.schema.yaml | 19 +- .../conversations/common_attributes.gen.ts | 2 +- .../common_attributes.schema.yaml | 8 +- .../impl/schemas/index.ts | 3 - .../prompts/bulk_crud_prompts_route.gen.ts | 135 + .../bulk_crud_prompts_route.schema.yaml | 259 + .../schemas/prompts/crud_prompts_route.gen.ts | 170 - .../prompts/crud_prompts_route.schema.yaml | 240 - .../schemas/prompts/find_prompts_route.gen.ts | 4 +- .../prompts/find_prompts_route.schema.yaml | 4 +- .../server/ai_assistant_service/index.test.ts | 24 +- .../server/ai_assistant_service/index.ts | 136 +- ...anonymization_fields_configuration_type.ts | 71 + .../find_anonymization_fields.ts | 94 + .../helpers.ts | 80 + .../anonymization_fields_data_client/index.ts | 131 + .../anonymization_fields_data_client/types.ts | 35 + .../append_conversation_messages.ts | 2 - .../conversations_configuration_type.ts} | 43 +- .../conversations_data_writer.ts | 23 +- .../create_conversation.test.ts | 18 +- .../create_conversation.ts | 19 +- .../get_conversation.ts | 23 +- .../server/conversations_data_client/index.ts | 2 +- .../conversations_data_client/transforms.ts | 9 +- .../server/conversations_data_client/types.ts | 9 +- .../update_conversation.test.ts | 10 +- .../update_conversation.ts | 5 - .../data_client/documents_data_writer.test.ts | 202 + .../lib/data_client/documents_data_writer.ts | 312 ++ .../server/lib/data_client/helper.ts | 23 + .../elastic_assistant/server/plugin.ts | 4 - .../server/promts_data_client/find_prompts.ts | 94 + .../server/promts_data_client/helpers.ts | 89 + .../server/promts_data_client/index.ts | 133 + .../prompts_configuration_type.ts | 86 + .../server/promts_data_client/types.ts | 35 + .../bulk_actions_route.ts | 80 +- .../routes/anonimization_fields/find_route.ts | 3 +- .../anonimization_fields/update_route.ts | 79 - .../routes/prompts/bulk_actions_route.ts | 215 + .../server/routes/prompts/create_route.ts | 63 - .../server/routes/prompts/delete_route.ts | 70 - .../server/routes/prompts/find_route.ts | 2 +- .../server/routes/prompts/update_route.ts | 79 - .../server/routes/register_routes.ts | 14 +- .../server/routes/request_context_factory.ts | 30 +- .../find_user_conversations_route.ts | 2 +- ...ssistant_anonymization_fields_so_client.ts | 244 - .../ai_assistant_prompts_so_client.ts | 218 - ...tic_assistant_anonymization_fields_type.ts | 145 - .../elastic_assistant_prompts_type.ts | 164 - .../server/saved_object/index.ts | 9 - .../plugins/elastic_assistant/server/types.ts | 16 +- 61 files changed, 4489 insertions(+), 3967 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen.ts create mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.schema.yaml delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts delete mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml create mode 100644 x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/anonymization_fields_configuration_type.ts create mode 100644 x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/find_anonymization_fields.ts create mode 100644 x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/helpers.ts create mode 100644 x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/types.ts rename x-pack/plugins/elastic_assistant/server/{ai_assistant_service/conversation_configuration_type.ts => conversations_data_client/conversations_configuration_type.ts} (71%) create mode 100644 x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/data_client/helper.ts create mode 100644 x-pack/plugins/elastic_assistant/server/promts_data_client/find_prompts.ts create mode 100644 x-pack/plugins/elastic_assistant/server/promts_data_client/helpers.ts create mode 100644 x-pack/plugins/elastic_assistant/server/promts_data_client/index.ts create mode 100644 x-pack/plugins/elastic_assistant/server/promts_data_client/prompts_configuration_type.ts create mode 100644 x-pack/plugins/elastic_assistant/server/promts_data_client/types.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/bulk_actions_route.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts delete mode 100644 x-pack/plugins/elastic_assistant/server/saved_object/index.ts diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index d1994f9786d54..56308a980cc56 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -974,28 +974,5 @@ ], "cloud-security-posture-settings": [ "rules" - ], - "elastic-ai-assistant-prompts": [ - "content", - "created_at", - "created_by", - "id", - "is_default", - "is_new_conversation_default", - "is_shared", - "name", - "prompt_type", - "updated_at", - "updated_by" - ], - "elastic-ai-assistant-anonymization-fields": [ - "created_at", - "created_by", - "default_allow", - "default_allow_replacement", - "field_id", - "id", - "updated_at", - "updated_by" ] } diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 8b334156bf3d2..3c7a34a02a956 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1,941 +1,1052 @@ { - "core-usage-stats": { - "dynamic": false, - "properties": {} - }, - "legacy-url-alias": { - "dynamic": false, - "properties": { - "sourceId": { - "type": "keyword" - }, - "targetNamespace": { - "type": "keyword" - }, - "targetType": { - "type": "keyword" - }, - "targetId": { - "type": "keyword" - }, - "resolveCounter": { - "type": "long" - }, - "disabled": { - "type": "boolean" - } - } - }, - "config": { - "dynamic": false, - "properties": { - "buildNum": { - "type": "keyword" - } - } - }, - "config-global": { - "dynamic": false, - "properties": { - "buildNum": { - "type": "keyword" - } - } - }, - "url": { - "dynamic": false, - "properties": { - "slug": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "accessDate": { - "type": "date" - }, - "createDate": { - "type": "date" - } - } - }, - "usage-counters": { - "dynamic": false, - "properties": { - "domainId": { - "type": "keyword" - } - } - }, - "task": { - "dynamic": false, - "properties": { - "taskType": { - "type": "keyword" - }, - "scheduledAt": { - "type": "date" - }, - "runAt": { - "type": "date" - }, - "retryAt": { - "type": "date" - }, - "enabled": { - "type": "boolean" - }, - "schedule": { - "properties": { - "interval": { - "type": "keyword" - } - } - }, - "attempts": { - "type": "integer" - }, - "status": { - "type": "keyword" - }, - "scope": { - "type": "keyword" - }, - "ownerId": { - "type": "keyword" - } - } - }, - "guided-onboarding-guide-state": { - "dynamic": false, - "properties": { - "guideId": { - "type": "keyword" - }, - "isActive": { - "type": "boolean" - } - } - }, - "guided-onboarding-plugin-state": { - "dynamic": false, - "properties": {} - }, - "ui-metric": { - "properties": { - "count": { - "type": "integer" - } - } - }, - "application_usage_totals": { - "dynamic": false, - "properties": {} - }, - "application_usage_daily": { - "dynamic": false, - "properties": { - "timestamp": { - "type": "date" - } - } - }, - "event_loop_delays_daily": { - "dynamic": false, - "properties": { - "lastUpdatedAt": { - "type": "date" - } - } - }, - "index-pattern": { + "action": { "dynamic": false, "properties": { - "title": { - "type": "text" - }, - "type": { + "actionTypeId": { "type": "keyword" }, "name": { - "type": "text", "fields": { "keyword": { "type": "keyword" } - } - } - } - }, - "sample-data-telemetry": { - "properties": { - "installCount": { - "type": "long" - }, - "unInstallCount": { - "type": "long" - } - } - }, - "space": { - "dynamic": false, - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 2048 - } - } + }, + "type": "text" } } }, - "spaces-usage-stats": { + "action_task_params": { "dynamic": false, "properties": {} }, - "exception-list-agnostic": { + "alert": { + "dynamic": false, "properties": { - "_tags": { - "type": "keyword" - }, - "created_at": { - "type": "keyword" - }, - "created_by": { - "type": "keyword" - }, - "description": { - "type": "keyword" - }, - "immutable": { - "type": "boolean" - }, - "list_id": { - "type": "keyword" - }, - "list_type": { - "type": "keyword" - }, - "meta": { - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "type": "text" - } - }, - "type": "keyword" - }, - "tags": { - "fields": { - "text": { - "type": "text" - } - }, - "type": "keyword" - }, - "tie_breaker_id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "updated_by": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "comments": { - "properties": { - "comment": { - "type": "keyword" - }, - "created_at": { - "type": "keyword" - }, - "created_by": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "updated_at": { - "type": "keyword" - }, - "updated_by": { - "type": "keyword" - } - } - }, - "entries": { + "actions": { + "dynamic": false, "properties": { - "entries": { - "properties": { - "field": { - "type": "keyword" - }, - "operator": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "value": { - "fields": { - "text": { - "type": "text" - } - }, - "type": "keyword" - } - } - }, - "field": { - "type": "keyword" - }, - "list": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "operator": { + "actionRef": { "type": "keyword" }, - "type": { + "actionTypeId": { "type": "keyword" }, - "value": { - "fields": { - "text": { - "type": "text" - } - }, + "group": { "type": "keyword" } - } - }, - "expire_time": { - "type": "date" - }, - "item_id": { - "type": "keyword" - }, - "os_types": { - "type": "keyword" - } - } - }, - "exception-list": { - "properties": { - "_tags": { - "type": "keyword" - }, - "created_at": { - "type": "keyword" - }, - "created_by": { - "type": "keyword" - }, - "description": { - "type": "keyword" - }, - "immutable": { - "type": "boolean" - }, - "list_id": { - "type": "keyword" - }, - "list_type": { - "type": "keyword" - }, - "meta": { - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "type": "text" - } }, - "type": "keyword" - }, - "tags": { - "fields": { - "text": { - "type": "text" - } - }, - "type": "keyword" - }, - "tie_breaker_id": { - "type": "keyword" - }, - "type": { - "type": "keyword" + "type": "nested" }, - "updated_by": { + "alertTypeId": { "type": "keyword" }, - "version": { + "consumer": { "type": "keyword" }, - "comments": { - "properties": { - "comment": { - "type": "keyword" - }, - "created_at": { - "type": "keyword" - }, - "created_by": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "updated_at": { - "type": "keyword" - }, - "updated_by": { - "type": "keyword" - } - } + "createdAt": { + "type": "date" }, - "entries": { + "createdBy": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "executionStatus": { "properties": { - "entries": { + "error": { "properties": { - "field": { - "type": "keyword" - }, - "operator": { - "type": "keyword" - }, - "type": { + "message": { "type": "keyword" }, - "value": { - "fields": { - "text": { - "type": "text" - } - }, + "reason": { "type": "keyword" } } }, - "field": { + "lastDuration": { + "type": "long" + }, + "lastExecutionDate": { + "type": "date" + }, + "numberOfTriggeredActions": { + "type": "long" + }, + "status": { "type": "keyword" }, - "list": { + "warning": { "properties": { - "id": { + "message": { "type": "keyword" }, - "type": { + "reason": { "type": "keyword" } } + } + } + }, + "lastRun": { + "properties": { + "alertsCount": { + "properties": { + "active": { + "type": "float" + }, + "ignored": { + "type": "float" + }, + "new": { + "type": "float" + }, + "recovered": { + "type": "float" + } + } }, - "operator": { + "outcome": { "type": "keyword" }, - "type": { - "type": "keyword" + "outcomeOrder": { + "type": "float" + } + } + }, + "legacyId": { + "type": "keyword" + }, + "mapped_params": { + "properties": { + "risk_score": { + "type": "float" }, - "value": { - "fields": { - "text": { - "type": "text" - } - }, + "severity": { "type": "keyword" } } }, - "expire_time": { - "type": "date" + "monitoring": { + "properties": { + "run": { + "properties": { + "calculated_metrics": { + "properties": { + "p50": { + "type": "long" + }, + "p95": { + "type": "long" + }, + "p99": { + "type": "long" + }, + "success_ratio": { + "type": "float" + } + } + }, + "last_run": { + "properties": { + "metrics": { + "properties": { + "duration": { + "type": "long" + }, + "gap_duration_s": { + "type": "float" + }, + "total_alerts_created": { + "type": "float" + }, + "total_alerts_detected": { + "type": "float" + }, + "total_indexing_duration_ms": { + "type": "long" + }, + "total_search_duration_ms": { + "type": "long" + } + } + }, + "timestamp": { + "type": "date" + } + } + } + } + } + } }, - "item_id": { - "type": "keyword" + "muteAll": { + "type": "boolean" }, - "os_types": { + "mutedInstanceIds": { "type": "keyword" - } - } - }, - "telemetry": { - "dynamic": false, - "properties": {} - }, - "file": { - "dynamic": false, - "properties": { - "created": { - "type": "date" - }, - "Updated": { - "type": "date" }, "name": { + "fields": { + "keyword": { + "normalizer": "lowercase", + "type": "keyword" + } + }, "type": "text" }, - "user": { + "notifyWhen": { + "type": "keyword" + }, + "params": { + "ignore_above": 4096, "type": "flattened" }, - "Status": { - "type": "keyword" + "revision": { + "type": "long" }, - "mime_type": { - "type": "keyword" + "running": { + "type": "boolean" }, - "extension": { + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledTaskId": { "type": "keyword" }, - "size": { - "type": "long" + "snoozeSchedule": { + "properties": { + "duration": { + "type": "long" + }, + "id": { + "type": "keyword" + }, + "skipRecurrences": { + "format": "strict_date_time", + "type": "date" + } + }, + "type": "nested" }, - "Meta": { - "type": "flattened" + "tags": { + "type": "keyword" }, - "FileKind": { + "throttle": { "type": "keyword" }, - "hash": { - "dynamic": false, - "properties": {} + "updatedAt": { + "type": "date" + }, + "updatedBy": { + "type": "keyword" } } }, - "fileShare": { - "dynamic": false, + "api_key_pending_invalidation": { "properties": { - "created": { + "apiKeyId": { + "type": "keyword" + }, + "createdAt": { "type": "date" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" }, - "valid_until": { - "type": "long" + "kuery": { + "type": "text" }, - "token": { - "type": "keyword" + "serviceEnvironmentFilterEnabled": { + "type": "boolean" }, - "name": { - "type": "keyword" + "serviceNameFilterEnabled": { + "type": "boolean" + } + } + }, + "apm-indices": { + "dynamic": false, + "properties": {} + }, + "apm-server-schema": { + "properties": { + "schemaJson": { + "index": false, + "type": "text" } } }, - "action": { - "dynamic": false, + "apm-service-group": { "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } + "color": { + "type": "text" }, - "actionTypeId": { + "description": { + "type": "text" + }, + "groupName": { "type": "keyword" + }, + "kuery": { + "type": "text" } } }, - "action_task_params": { + "apm-telemetry": { "dynamic": false, "properties": {} }, - "connector_token": { + "app_search_telemetry": { "dynamic": false, - "properties": { - "connectorId": { - "type": "keyword" - }, - "tokenType": { - "type": "keyword" - } - } + "properties": {} }, - "query": { + "application_usage_daily": { "dynamic": false, "properties": { - "title": { - "type": "text" - }, - "description": { - "type": "text" + "timestamp": { + "type": "date" } } }, - "kql-telemetry": { + "application_usage_totals": { "dynamic": false, "properties": {} }, - "search-session": { + "canvas-element": { "dynamic": false, "properties": { - "sessionId": { - "type": "keyword" + "@created": { + "type": "date" }, - "created": { + "@timestamp": { "type": "date" }, - "realmType": { - "type": "keyword" + "content": { + "type": "text" }, - "realmName": { - "type": "keyword" + "help": { + "type": "text" }, - "username": { - "type": "keyword" + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" } } }, - "search-telemetry": { + "canvas-workpad": { "dynamic": false, - "properties": {} - }, - "file-upload-usage-collection-telemetry": { "properties": { - "file_upload": { - "properties": { - "index_creation_count": { - "type": "long" + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" } - } + }, + "type": "text" } } }, - "apm-indices": { + "canvas-workpad-template": { "dynamic": false, - "properties": {} - }, - "tag": { "properties": { - "name": { + "help": { + "fields": { + "keyword": { + "type": "keyword" + } + }, "type": "text" }, - "description": { + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, "type": "text" }, - "color": { + "tags": { + "fields": { + "keyword": { + "type": "keyword" + } + }, "type": "text" + }, + "template_key": { + "type": "keyword" } } }, - "alert": { + "cases": { "dynamic": false, "properties": { - "enabled": { - "type": "boolean" + "assignees": { + "properties": { + "uid": { + "type": "keyword" + } + } }, - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase" + "category": { + "type": "keyword" + }, + "closed_at": { + "type": "date" + }, + "closed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "profile_uid": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "connector": { + "properties": { + "fields": { + "properties": { + "key": { + "type": "text" + }, + "value": { + "type": "text" + } + } + }, + "name": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "profile_uid": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "customFields": { + "properties": { + "key": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "boolean": { + "ignore_malformed": true, + "type": "boolean" + }, + "date": { + "ignore_malformed": true, + "type": "date" + }, + "ip": { + "ignore_malformed": true, + "type": "ip" + }, + "number": { + "ignore_malformed": true, + "type": "long" + }, + "string": { + "type": "text" + } + }, + "type": "keyword" + } + }, + "type": "nested" + }, + "description": { + "type": "text" + }, + "duration": { + "type": "unsigned_long" + }, + "external_service": { + "properties": { + "connector_name": { + "type": "keyword" + }, + "external_id": { + "type": "keyword" + }, + "external_title": { + "type": "text" + }, + "external_url": { + "type": "text" + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "profile_uid": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } } } }, - "tags": { - "type": "keyword" - }, - "alertTypeId": { + "owner": { "type": "keyword" }, - "schedule": { + "settings": { "properties": { - "interval": { - "type": "keyword" + "syncAlerts": { + "type": "boolean" } } }, - "consumer": { - "type": "keyword" + "severity": { + "type": "short" }, - "legacyId": { + "status": { + "type": "short" + }, + "tags": { "type": "keyword" }, - "actions": { - "dynamic": false, - "type": "nested", + "title": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "total_alerts": { + "type": "integer" + }, + "total_comments": { + "type": "integer" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { "properties": { - "group": { + "email": { "type": "keyword" }, - "actionRef": { + "full_name": { "type": "keyword" }, - "actionTypeId": { + "profile_uid": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-comments": { + "dynamic": false, + "properties": { + "actions": { + "properties": { + "type": { "type": "keyword" } } }, - "params": { - "type": "flattened", - "ignore_above": 4096 + "alertId": { + "type": "keyword" }, - "mapped_params": { + "comment": { + "type": "text" + }, + "created_at": { + "type": "date" + }, + "created_by": { "properties": { - "risk_score": { - "type": "float" - }, - "severity": { + "username": { "type": "keyword" } } }, - "scheduledTaskId": { + "externalReferenceAttachmentTypeId": { "type": "keyword" }, - "createdBy": { + "owner": { "type": "keyword" }, - "updatedBy": { + "persistableStateAttachmentTypeId": { "type": "keyword" }, - "createdAt": { + "pushed_at": { "type": "date" }, - "updatedAt": { - "type": "date" + "type": { + "type": "keyword" }, - "throttle": { + "updated_at": { + "type": "date" + } + } + }, + "cases-configure": { + "dynamic": false, + "properties": { + "closure_type": { "type": "keyword" }, - "notifyWhen": { + "created_at": { + "type": "date" + }, + "owner": { + "type": "keyword" + } + } + }, + "cases-connector-mappings": { + "dynamic": false, + "properties": { + "owner": { + "type": "keyword" + } + } + }, + "cases-telemetry": { + "dynamic": false, + "properties": {} + }, + "cases-user-actions": { + "dynamic": false, + "properties": { + "action": { "type": "keyword" }, - "muteAll": { - "type": "boolean" + "created_at": { + "type": "date" }, - "mutedInstanceIds": { + "created_by": { + "properties": { + "username": { + "type": "keyword" + } + } + }, + "owner": { "type": "keyword" }, - "monitoring": { + "payload": { + "dynamic": false, "properties": { - "run": { + "assignees": { "properties": { - "calculated_metrics": { - "properties": { - "p50": { - "type": "long" - }, - "p95": { - "type": "long" - }, - "p99": { - "type": "long" - }, - "success_ratio": { - "type": "float" - } - } + "uid": { + "type": "keyword" + } + } + }, + "comment": { + "properties": { + "externalReferenceAttachmentTypeId": { + "type": "keyword" }, - "last_run": { - "properties": { - "timestamp": { - "type": "date" - }, - "metrics": { - "properties": { - "duration": { - "type": "long" - }, - "total_search_duration_ms": { - "type": "long" - }, - "total_indexing_duration_ms": { - "type": "long" - }, - "total_alerts_detected": { - "type": "float" - }, - "total_alerts_created": { - "type": "float" - }, - "gap_duration_s": { - "type": "float" - } - } - } - } + "persistableStateAttachmentTypeId": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "connector": { + "properties": { + "type": { + "type": "keyword" } } } } }, - "revision": { - "type": "long" + "type": { + "type": "keyword" + } + } + }, + "cloud-security-posture-settings": { + "dynamic": false, + "properties": {} + }, + "config": { + "dynamic": false, + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "config-global": { + "dynamic": false, + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "connector_token": { + "dynamic": false, + "properties": { + "connectorId": { + "type": "keyword" }, - "snoozeSchedule": { - "type": "nested", + "tokenType": { + "type": "keyword" + } + } + }, + "core-usage-stats": { + "dynamic": false, + "properties": {} + }, + "csp-rule-template": { + "dynamic": false, + "properties": { + "metadata": { "properties": { + "benchmark": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "posture_type": { + "type": "keyword" + }, + "rule_number": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + }, + "type": "object" + }, "id": { "type": "keyword" }, - "duration": { - "type": "long" + "name": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" }, - "skipRecurrences": { - "type": "date", - "format": "strict_date_time" + "section": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "version": { + "type": "keyword" } - } - }, - "executionStatus": { + }, + "type": "object" + } + } + }, + "dashboard": { + "properties": { + "controlGroupInput": { "properties": { - "numberOfTriggeredActions": { - "type": "long" - }, - "status": { + "chainingSystem": { + "doc_values": false, + "index": false, "type": "keyword" }, - "lastExecutionDate": { - "type": "date" - }, - "lastDuration": { - "type": "long" + "controlStyle": { + "doc_values": false, + "index": false, + "type": "keyword" }, - "error": { - "properties": { - "reason": { - "type": "keyword" - }, - "message": { - "type": "keyword" - } - } + "ignoreParentSettingsJSON": { + "index": false, + "type": "text" }, - "warning": { - "properties": { - "reason": { - "type": "keyword" - }, - "message": { - "type": "keyword" - } - } + "panelsJSON": { + "index": false, + "type": "text" } } }, - "lastRun": { + "description": { + "type": "text" + }, + "hits": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "kibanaSavedObjectMeta": { "properties": { - "outcome": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "optionsJSON": { + "index": false, + "type": "text" + }, + "panelsJSON": { + "index": false, + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "doc_values": false, + "index": false, "type": "keyword" }, - "outcomeOrder": { - "type": "float" + "pause": { + "doc_values": false, + "index": false, + "type": "boolean" }, - "alertsCount": { - "properties": { - "active": { - "type": "float" - }, - "new": { - "type": "float" - }, - "recovered": { - "type": "float" - }, - "ignored": { - "type": "float" - } - } + "section": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "value": { + "doc_values": false, + "index": false, + "type": "integer" } } }, - "running": { + "timeFrom": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "timeRestore": { + "doc_values": false, + "index": false, "type": "boolean" - } - } - }, - "api_key_pending_invalidation": { - "properties": { - "apiKeyId": { + }, + "timeTo": { + "doc_values": false, + "index": false, "type": "keyword" }, - "createdAt": { - "type": "date" + "title": { + "type": "text" + }, + "version": { + "type": "integer" } } }, - "rules-settings": { + "endpoint:user-artifact-manifest": { "dynamic": false, "properties": { - "flapping": { - "properties": {} + "artifacts": { + "type": "nested" + }, + "schemaVersion": { + "type": "keyword" } } }, - "maintenance-window": { + "enterprise_search_telemetry": { "dynamic": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "events": { - "type": "date_range", - "format": "epoch_millis||strict_date_optional_time" - } - } + "properties": {} }, - "graph-workspace": { + "epm-packages": { "properties": { - "description": { - "type": "text" + "es_index_patterns": { + "dynamic": false, + "properties": {} }, - "kibanaSavedObjectMeta": { + "experimental_data_stream_features": { "properties": { - "searchSourceJSON": { - "type": "text" + "data_stream": { + "type": "keyword" + }, + "features": { + "dynamic": false, + "properties": { + "synthetic_source": { + "type": "boolean" + }, + "tsdb": { + "type": "boolean" + } + }, + "type": "nested" } - } + }, + "type": "nested" + }, + "install_format_schema_version": { + "type": "version" + }, + "install_source": { + "type": "keyword" + }, + "install_started_at": { + "type": "date" + }, + "install_status": { + "type": "keyword" + }, + "install_version": { + "type": "keyword" + }, + "installed_es": { + "properties": { + "deferred": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + }, + "type": "nested" + }, + "installed_kibana": { + "dynamic": false, + "properties": {} + }, + "installed_kibana_space_id": { + "type": "keyword" + }, + "internal": { + "type": "boolean" }, - "numLinks": { - "type": "integer" + "keep_policies_up_to_date": { + "index": false, + "type": "boolean" }, - "numVertices": { - "type": "integer" + "latest_install_failed_attempts": { + "enabled": false, + "type": "object" }, - "title": { - "type": "text" + "name": { + "type": "keyword" }, - "version": { - "type": "integer" + "package_assets": { + "dynamic": false, + "properties": {} }, - "wsState": { - "type": "text" + "verification_key_id": { + "type": "keyword" }, - "legacyIndexPatternRef": { - "type": "text", - "index": false + "verification_status": { + "type": "keyword" + }, + "version": { + "type": "keyword" } } }, - "search": { - "dynamic": false, + "epm-packages-assets": { "properties": { - "title": { - "type": "text" + "asset_path": { + "type": "keyword" }, - "description": { + "data_base64": { + "type": "binary" + }, + "data_utf8": { + "index": false, "type": "text" + }, + "install_source": { + "type": "keyword" + }, + "media_type": { + "type": "keyword" + }, + "package_name": { + "type": "keyword" + }, + "package_version": { + "type": "keyword" } } }, - "visualization": { + "event-annotation-group": { "dynamic": false, "properties": { "description": { @@ -943,617 +1054,543 @@ }, "title": { "type": "text" - }, - "version": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": {} } } }, - "event-annotation-group": { + "event_loop_delays_daily": { "dynamic": false, "properties": { - "title": { - "type": "text" - }, - "description": { - "type": "text" + "lastUpdatedAt": { + "type": "date" } } }, - "dashboard": { + "exception-list": { "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer", - "index": false, - "doc_values": false + "_tags": { + "type": "keyword" }, - "kibanaSavedObjectMeta": { + "comments": { "properties": { - "searchSourceJSON": { - "type": "text", - "index": false + "comment": { + "type": "keyword" + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "updated_at": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" } } }, - "optionsJSON": { - "type": "text", - "index": false + "created_at": { + "type": "keyword" }, - "panelsJSON": { - "type": "text", - "index": false + "created_by": { + "type": "keyword" }, - "refreshInterval": { + "description": { + "type": "keyword" + }, + "entries": { "properties": { - "display": { - "type": "keyword", - "index": false, - "doc_values": false - }, - "pause": { - "type": "boolean", - "index": false, - "doc_values": false + "entries": { + "properties": { + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + } + } }, - "section": { - "type": "integer", - "index": false, - "doc_values": false + "field": { + "type": "keyword" }, - "value": { - "type": "integer", - "index": false, - "doc_values": false - } - } - }, - "controlGroupInput": { - "properties": { - "controlStyle": { - "type": "keyword", - "index": false, - "doc_values": false + "list": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } }, - "chainingSystem": { - "type": "keyword", - "index": false, - "doc_values": false + "operator": { + "type": "keyword" }, - "panelsJSON": { - "type": "text", - "index": false + "type": { + "type": "keyword" }, - "ignoreParentSettingsJSON": { - "type": "text", - "index": false + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" } } }, - "timeFrom": { - "type": "keyword", - "index": false, - "doc_values": false - }, - "timeRestore": { - "type": "boolean", - "index": false, - "doc_values": false - }, - "timeTo": { - "type": "keyword", - "index": false, - "doc_values": false - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "lens": { - "properties": { - "title": { - "type": "text" - }, - "description": { - "type": "text" + "expire_time": { + "type": "date" }, - "visualizationType": { - "type": "keyword" + "immutable": { + "type": "boolean" }, - "state": { - "dynamic": false, - "properties": {} - } - } - }, - "lens-ui-telemetry": { - "properties": { - "name": { + "item_id": { "type": "keyword" }, - "type": { + "list_id": { "type": "keyword" }, - "date": { - "type": "date" - }, - "count": { - "type": "integer" - } - } - }, - "cases-comments": { - "dynamic": false, - "properties": { - "comment": { - "type": "text" - }, - "owner": { + "list_type": { "type": "keyword" }, - "type": { + "meta": { "type": "keyword" }, - "actions": { - "properties": { - "type": { - "type": "keyword" + "name": { + "fields": { + "text": { + "type": "text" } - } - }, - "alertId": { + }, "type": "keyword" }, - "created_at": { - "type": "date" - }, - "created_by": { - "properties": { - "username": { - "type": "keyword" - } - } - }, - "externalReferenceAttachmentTypeId": { + "os_types": { "type": "keyword" }, - "persistableStateAttachmentTypeId": { + "tags": { + "fields": { + "text": { + "type": "text" + } + }, "type": "keyword" }, - "pushed_at": { - "type": "date" + "tie_breaker_id": { + "type": "keyword" }, - "updated_at": { - "type": "date" - } - } - }, - "cases-configure": { - "dynamic": false, - "properties": { - "created_at": { - "type": "date" + "type": { + "type": "keyword" }, - "closure_type": { + "updated_by": { "type": "keyword" }, - "owner": { + "version": { "type": "keyword" } } }, - "cases-connector-mappings": { - "dynamic": false, + "exception-list-agnostic": { "properties": { - "owner": { + "_tags": { "type": "keyword" - } - } - }, - "cases": { - "dynamic": false, - "properties": { - "assignees": { - "properties": { - "uid": { - "type": "keyword" - } - } - }, - "closed_at": { - "type": "date" }, - "closed_by": { + "comments": { "properties": { - "username": { - "type": "keyword" - }, - "full_name": { + "comment": { "type": "keyword" }, - "email": { + "created_at": { "type": "keyword" }, - "profile_uid": { - "type": "keyword" - } - } - }, - "created_at": { - "type": "date" - }, - "created_by": { - "properties": { - "username": { + "created_by": { "type": "keyword" }, - "full_name": { + "id": { "type": "keyword" }, - "email": { + "updated_at": { "type": "keyword" }, - "profile_uid": { + "updated_by": { "type": "keyword" } } }, - "duration": { - "type": "unsigned_long" + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" }, "description": { - "type": "text" + "type": "keyword" }, - "connector": { + "entries": { "properties": { - "name": { - "type": "text" - }, - "type": { - "type": "keyword" - }, - "fields": { + "entries": { "properties": { - "key": { - "type": "text" + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" }, "value": { - "type": "text" + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" } } - } - } - }, - "external_service": { - "properties": { - "pushed_at": { - "type": "date" }, - "pushed_by": { + "field": { + "type": "keyword" + }, + "list": { "properties": { - "username": { - "type": "keyword" - }, - "full_name": { - "type": "keyword" - }, - "email": { + "id": { "type": "keyword" }, - "profile_uid": { + "type": { "type": "keyword" } } }, - "connector_name": { + "operator": { "type": "keyword" }, - "external_id": { + "type": { "type": "keyword" }, - "external_title": { - "type": "text" - }, - "external_url": { - "type": "text" + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" } } }, - "owner": { + "expire_time": { + "type": "date" + }, + "immutable": { + "type": "boolean" + }, + "item_id": { "type": "keyword" }, - "title": { - "type": "text", + "list_id": { + "type": "keyword" + }, + "list_type": { + "type": "keyword" + }, + "meta": { + "type": "keyword" + }, + "name": { "fields": { - "keyword": { - "type": "keyword" + "text": { + "type": "text" } - } + }, + "type": "keyword" }, - "status": { - "type": "short" + "os_types": { + "type": "keyword" }, "tags": { + "fields": { + "text": { + "type": "text" + } + }, "type": "keyword" }, - "updated_at": { - "type": "date" + "tie_breaker_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" }, "updated_by": { - "properties": { - "username": { - "type": "keyword" - }, - "full_name": { - "type": "keyword" - }, - "email": { - "type": "keyword" - }, - "profile_uid": { - "type": "keyword" - } - } + "type": "keyword" }, - "settings": { - "properties": { - "syncAlerts": { - "type": "boolean" - } - } + "version": { + "type": "keyword" + } + } + }, + "file": { + "dynamic": false, + "properties": { + "FileKind": { + "type": "keyword" }, - "severity": { - "type": "short" + "Meta": { + "type": "flattened" }, - "total_alerts": { - "type": "integer" + "Status": { + "type": "keyword" }, - "total_comments": { - "type": "integer" + "Updated": { + "type": "date" }, - "category": { + "created": { + "type": "date" + }, + "extension": { "type": "keyword" }, - "customFields": { - "type": "nested", + "hash": { + "dynamic": false, + "properties": {} + }, + "mime_type": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "size": { + "type": "long" + }, + "user": { + "type": "flattened" + } + } + }, + "file-upload-usage-collection-telemetry": { + "properties": { + "file_upload": { "properties": { - "key": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "value": { - "type": "keyword", - "fields": { - "number": { - "type": "long", - "ignore_malformed": true - }, - "boolean": { - "type": "boolean", - "ignore_malformed": true - }, - "string": { - "type": "text" - }, - "date": { - "type": "date", - "ignore_malformed": true - }, - "ip": { - "type": "ip", - "ignore_malformed": true - } - } + "index_creation_count": { + "type": "long" } } } } }, - "cases-user-actions": { + "fileShare": { "dynamic": false, "properties": { - "action": { + "created": { + "type": "date" + }, + "name": { "type": "keyword" }, - "created_at": { - "type": "date" + "token": { + "type": "keyword" }, - "created_by": { - "properties": { - "username": { - "type": "keyword" - } - } + "valid_until": { + "type": "long" + } + } + }, + "fleet-fleet-server-host": { + "properties": { + "host_urls": { + "index": false, + "type": "keyword" }, - "payload": { - "dynamic": false, - "properties": { - "connector": { - "properties": { - "type": { - "type": "keyword" - } - } - }, - "comment": { - "properties": { - "type": { - "type": "keyword" - }, - "externalReferenceAttachmentTypeId": { - "type": "keyword" - }, - "persistableStateAttachmentTypeId": { - "type": "keyword" - } - } - }, - "assignees": { - "properties": { - "uid": { - "type": "keyword" - } - } - } - } + "is_default": { + "type": "boolean" }, - "owner": { + "is_internal": { + "index": false, + "type": "boolean" + }, + "is_preconfigured": { + "type": "boolean" + }, + "name": { "type": "keyword" }, - "type": { + "proxy_id": { "type": "keyword" } } }, - "cases-telemetry": { + "fleet-message-signing-keys": { "dynamic": false, "properties": {} }, - "metrics-data-source": { - "dynamic": false, - "properties": {} + "fleet-preconfiguration-deletion-record": { + "properties": { + "id": { + "type": "keyword" + } + } }, - "links": { - "dynamic": false, + "fleet-proxy": { "properties": { - "title": { - "type": "text" + "certificate": { + "index": false, + "type": "keyword" }, - "description": { + "certificate_authorities": { + "index": false, + "type": "keyword" + }, + "certificate_key": { + "index": false, + "type": "keyword" + }, + "is_preconfigured": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "proxy_headers": { + "index": false, "type": "text" }, - "links": { - "dynamic": false, - "properties": {} + "url": { + "index": false, + "type": "keyword" } } }, - "canvas-element": { + "fleet-uninstall-tokens": { "dynamic": false, "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" + "policy_id": { + "type": "keyword" + }, + "token_plain": { + "type": "keyword" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" } } }, - "help": { + "legacyIndexPatternRef": { + "index": false, "type": "text" }, - "content": { - "type": "text" + "numLinks": { + "type": "integer" }, - "image": { + "numVertices": { + "type": "integer" + }, + "title": { "type": "text" }, - "@timestamp": { - "type": "date" + "version": { + "type": "integer" }, - "@created": { - "type": "date" + "wsState": { + "type": "text" } } }, - "canvas-workpad": { + "guided-onboarding-guide-state": { "dynamic": false, "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" + "guideId": { + "type": "keyword" }, - "@created": { - "type": "date" + "isActive": { + "type": "boolean" } } }, - "canvas-workpad-template": { + "guided-onboarding-plugin-state": { + "dynamic": false, + "properties": {} + }, + "index-pattern": { "dynamic": false, "properties": { "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "help": { - "type": "text", "fields": { "keyword": { "type": "keyword" } - } + }, + "type": "text" }, - "tags": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } + "title": { + "type": "text" }, - "template_key": { + "type": { "type": "keyword" } } }, - "ingest_manager_settings": { + "infrastructure-monitoring-log-view": { + "dynamic": false, "properties": { - "fleet_server_hosts": { - "type": "keyword" - }, - "has_seen_add_data_notice": { - "type": "boolean", - "index": false - }, - "prerelease_integrations_enabled": { - "type": "boolean" - }, - "secret_storage_requirements_met": { - "type": "boolean" - }, - "output_secret_storage_requirements_met": { - "type": "boolean" + "name": { + "type": "text" } } }, + "infrastructure-ui-source": { + "dynamic": false, + "properties": {} + }, "ingest-agent-policies": { "properties": { - "name": { - "type": "keyword" + "agent_features": { + "properties": { + "enabled": { + "type": "boolean" + }, + "name": { + "type": "keyword" + } + } }, - "schema_version": { - "type": "version" + "data_output_id": { + "type": "keyword" }, "description": { "type": "text" }, - "namespace": { + "download_source_id": { "type": "keyword" }, - "is_managed": { - "type": "boolean" + "fleet_server_host_id": { + "type": "keyword" + }, + "inactivity_timeout": { + "type": "integer" }, "is_default": { "type": "boolean" @@ -1561,179 +1598,120 @@ "is_default_fleet_server": { "type": "boolean" }, - "status": { - "type": "keyword" - }, - "unenroll_timeout": { - "type": "integer" - }, - "inactivity_timeout": { - "type": "integer" - }, - "updated_at": { - "type": "date" + "is_managed": { + "type": "boolean" }, - "updated_by": { + "is_preconfigured": { "type": "keyword" }, - "revision": { - "type": "integer" - }, - "monitoring_enabled": { - "type": "keyword", - "index": false + "is_protected": { + "type": "boolean" }, - "is_preconfigured": { - "type": "keyword" + "keep_monitoring_alive": { + "type": "boolean" }, - "data_output_id": { + "monitoring_enabled": { + "index": false, "type": "keyword" }, "monitoring_output_id": { "type": "keyword" }, - "download_source_id": { + "name": { "type": "keyword" }, - "fleet_server_host_id": { + "namespace": { "type": "keyword" }, - "agent_features": { - "properties": { - "name": { - "type": "keyword" - }, - "enabled": { - "type": "boolean" - } - } + "overrides": { + "index": false, + "type": "flattened" }, - "is_protected": { - "type": "boolean" + "revision": { + "type": "integer" }, - "overrides": { - "type": "flattened", - "index": false + "schema_version": { + "type": "version" }, - "keep_monitoring_alive": { - "type": "boolean" + "status": { + "type": "keyword" + }, + "unenroll_timeout": { + "type": "integer" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" } } }, - "ingest-outputs": { + "ingest-download-sources": { "properties": { - "output_id": { - "type": "keyword", - "index": false - }, - "name": { - "type": "keyword" - }, - "type": { + "host": { "type": "keyword" }, "is_default": { "type": "boolean" }, - "is_default_monitoring": { - "type": "boolean" - }, - "hosts": { + "name": { "type": "keyword" }, - "ca_sha256": { - "type": "keyword", - "index": false - }, - "ca_trusted_fingerprint": { - "type": "keyword", - "index": false - }, - "service_token": { - "type": "keyword", - "index": false - }, - "config": { - "type": "flattened" - }, - "config_yaml": { - "type": "text" - }, - "is_preconfigured": { - "type": "boolean", - "index": false - }, - "is_internal": { - "type": "boolean", - "index": false - }, - "ssl": { - "type": "binary" - }, "proxy_id": { "type": "keyword" }, - "shipper": { - "dynamic": false, - "properties": {} - }, + "source_id": { + "index": false, + "type": "keyword" + } + } + }, + "ingest-outputs": { + "properties": { "allow_edit": { "enabled": false }, - "version": { + "auth_type": { "type": "keyword" }, - "key": { - "type": "keyword" + "broker_ack_reliability": { + "type": "text" }, - "compression": { - "type": "keyword" + "broker_buffer_size": { + "type": "integer" }, - "compression_level": { + "broker_timeout": { "type": "integer" }, - "client_id": { + "ca_sha256": { + "index": false, "type": "keyword" }, - "auth_type": { + "ca_trusted_fingerprint": { + "index": false, "type": "keyword" }, - "connection_type": { - "type": "keyword" + "channel_buffer_size": { + "type": "integer" }, - "username": { + "client_id": { "type": "keyword" }, - "password": { - "type": "text", - "index": false + "compression": { + "type": "keyword" }, - "sasl": { - "dynamic": false, - "properties": { - "mechanism": { - "type": "text" - } - } + "compression_level": { + "type": "integer" }, - "partition": { - "type": "keyword" + "config": { + "type": "flattened" }, - "random": { - "dynamic": false, - "properties": { - "group_events": { - "type": "integer" - } - } + "config_yaml": { + "type": "text" }, - "round_robin": { - "dynamic": false, - "properties": { - "group_events": { - "type": "integer" - } - } + "connection_type": { + "type": "keyword" }, "hash": { "dynamic": false, @@ -1746,25 +1724,6 @@ } } }, - "topics": { - "dynamic": false, - "properties": { - "topic": { - "type": "keyword" - }, - "when": { - "dynamic": false, - "properties": { - "type": { - "type": "text" - }, - "condition": { - "type": "text" - } - } - } - } - }, "headers": { "dynamic": false, "properties": { @@ -1776,24 +1735,74 @@ } } }, - "timeout": { - "type": "integer" + "hosts": { + "type": "keyword" }, - "broker_timeout": { - "type": "integer" + "is_default": { + "type": "boolean" }, - "broker_ack_reliability": { + "is_default_monitoring": { + "type": "boolean" + }, + "is_internal": { + "index": false, + "type": "boolean" + }, + "is_preconfigured": { + "index": false, + "type": "boolean" + }, + "key": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "output_id": { + "index": false, + "type": "keyword" + }, + "partition": { + "type": "keyword" + }, + "password": { + "index": false, "type": "text" }, - "broker_buffer_size": { - "type": "integer" + "preset": { + "index": false, + "type": "keyword" }, - "required_acks": { - "type": "integer" + "proxy_id": { + "type": "keyword" }, - "channel_buffer_size": { + "random": { + "dynamic": false, + "properties": { + "group_events": { + "type": "integer" + } + } + }, + "required_acks": { "type": "integer" }, + "round_robin": { + "dynamic": false, + "properties": { + "group_events": { + "type": "integer" + } + } + }, + "sasl": { + "dynamic": false, + "properties": { + "mechanism": { + "type": "text" + } + } + }, "secrets": { "dynamic": false, "properties": { @@ -1805,6 +1814,14 @@ } } }, + "service_token": { + "dynamic": false, + "properties": { + "id": { + "type": "keyword" + } + } + }, "ssl": { "dynamic": false, "properties": { @@ -1817,1096 +1834,707 @@ } } } - }, - "service_token": { - "dynamic": false, - "properties": { - "id": { - "type": "keyword" - } - } } } }, - "preset": { - "type": "keyword", - "index": false - } - } - }, - "ingest-package-policies": { - "properties": { - "name": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "namespace": { - "type": "keyword" - }, - "enabled": { - "type": "boolean" - }, - "is_managed": { - "type": "boolean" - }, - "policy_id": { + "service_token": { + "index": false, "type": "keyword" }, - "package": { - "properties": { - "name": { - "type": "keyword" - }, - "title": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } - } - }, - "elasticsearch": { + "shipper": { "dynamic": false, "properties": {} }, - "vars": { - "type": "flattened" + "ssl": { + "type": "binary" }, - "inputs": { - "dynamic": false, - "properties": {} + "timeout": { + "type": "integer" }, - "secret_references": { + "topics": { + "dynamic": false, "properties": { - "id": { + "topic": { "type": "keyword" + }, + "when": { + "dynamic": false, + "properties": { + "condition": { + "type": "text" + }, + "type": { + "type": "text" + } + } } } }, - "revision": { - "type": "integer" - }, - "updated_at": { - "type": "date" - }, - "updated_by": { + "type": { "type": "keyword" }, - "created_at": { - "type": "date" + "username": { + "type": "keyword" }, - "created_by": { + "version": { "type": "keyword" } } }, - "epm-packages": { + "ingest-package-policies": { "properties": { - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "internal": { - "type": "boolean" - }, - "keep_policies_up_to_date": { - "type": "boolean", - "index": false - }, - "es_index_patterns": { - "dynamic": false, - "properties": {} - }, - "verification_status": { - "type": "keyword" + "created_at": { + "type": "date" }, - "verification_key_id": { + "created_by": { "type": "keyword" }, - "installed_es": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "deferred": { - "type": "boolean" - } - } - }, - "latest_install_failed_attempts": { - "type": "object", - "enabled": false + "description": { + "type": "text" }, - "installed_kibana": { + "elasticsearch": { "dynamic": false, "properties": {} }, - "installed_kibana_space_id": { - "type": "keyword" + "enabled": { + "type": "boolean" }, - "package_assets": { + "inputs": { "dynamic": false, "properties": {} }, - "install_started_at": { - "type": "date" - }, - "install_version": { - "type": "keyword" + "is_managed": { + "type": "boolean" }, - "install_status": { + "name": { "type": "keyword" }, - "install_source": { + "namespace": { "type": "keyword" }, - "install_format_schema_version": { - "type": "version" - }, - "experimental_data_stream_features": { - "type": "nested", + "package": { "properties": { - "data_stream": { + "name": { "type": "keyword" }, - "features": { - "type": "nested", - "dynamic": false, - "properties": { - "synthetic_source": { - "type": "boolean" - }, - "tsdb": { - "type": "boolean" - } - } + "title": { + "type": "keyword" + }, + "version": { + "type": "keyword" } } - } - } - }, - "epm-packages-assets": { - "properties": { - "package_name": { - "type": "keyword" - }, - "package_version": { - "type": "keyword" - }, - "install_source": { - "type": "keyword" - }, - "asset_path": { - "type": "keyword" - }, - "media_type": { - "type": "keyword" - }, - "data_utf8": { - "type": "text", - "index": false - }, - "data_base64": { - "type": "binary" - } - } - }, - "fleet-preconfiguration-deletion-record": { - "properties": { - "id": { - "type": "keyword" - } - } - }, - "ingest-download-sources": { - "properties": { - "source_id": { - "type": "keyword", - "index": false - }, - "name": { - "type": "keyword" - }, - "is_default": { - "type": "boolean" - }, - "host": { - "type": "keyword" }, - "proxy_id": { - "type": "keyword" - } - } - }, - "fleet-fleet-server-host": { - "properties": { - "name": { + "policy_id": { "type": "keyword" }, - "is_default": { - "type": "boolean" - }, - "is_internal": { - "type": "boolean", - "index": false + "revision": { + "type": "integer" }, - "host_urls": { - "type": "keyword", - "index": false + "secret_references": { + "properties": { + "id": { + "type": "keyword" + } + } }, - "is_preconfigured": { - "type": "boolean" + "updated_at": { + "type": "date" }, - "proxy_id": { + "updated_by": { "type": "keyword" + }, + "vars": { + "type": "flattened" } } }, - "fleet-proxy": { + "ingest_manager_settings": { "properties": { - "name": { + "fleet_server_hosts": { "type": "keyword" }, - "url": { - "type": "keyword", - "index": false - }, - "proxy_headers": { - "type": "text", - "index": false - }, - "certificate_authorities": { - "type": "keyword", - "index": false + "has_seen_add_data_notice": { + "index": false, + "type": "boolean" }, - "certificate": { - "type": "keyword", - "index": false + "output_secret_storage_requirements_met": { + "type": "boolean" }, - "certificate_key": { - "type": "keyword", - "index": false + "prerelease_integrations_enabled": { + "type": "boolean" }, - "is_preconfigured": { + "secret_storage_requirements_met": { "type": "boolean" } } }, - "fleet-message-signing-keys": { + "inventory-view": { "dynamic": false, "properties": {} }, - "fleet-uninstall-tokens": { + "kql-telemetry": { "dynamic": false, - "properties": { - "policy_id": { - "type": "keyword" - }, - "token_plain": { - "type": "keyword" - } - } + "properties": {} }, - "osquery-manager-usage-metric": { + "legacy-url-alias": { + "dynamic": false, "properties": { - "count": { - "type": "long" + "disabled": { + "type": "boolean" }, - "errors": { + "resolveCounter": { "type": "long" - } - } - }, - "osquery-saved-query": { - "dynamic": false, - "properties": { - "description": { - "type": "text" }, - "id": { + "sourceId": { "type": "keyword" }, - "query": { - "type": "text" - }, - "created_at": { - "type": "date" - }, - "created_by": { - "type": "text" - }, - "platform": { + "targetId": { "type": "keyword" }, - "version": { + "targetNamespace": { "type": "keyword" }, - "updated_at": { - "type": "date" - }, - "updated_by": { - "type": "text" - }, - "interval": { + "targetType": { "type": "keyword" - }, - "timeout": { - "type": "short" - }, - "ecs_mapping": { - "dynamic": false, - "properties": {} } } }, - "osquery-pack": { + "lens": { "properties": { "description": { "type": "text" }, - "name": { - "type": "text" + "state": { + "dynamic": false, + "properties": {} }, - "created_at": { - "type": "date" + "title": { + "type": "text" }, - "created_by": { + "visualizationType": { "type": "keyword" + } + } + }, + "lens-ui-telemetry": { + "properties": { + "count": { + "type": "integer" }, - "updated_at": { + "date": { "type": "date" }, - "updated_by": { + "name": { "type": "keyword" }, - "enabled": { - "type": "boolean" - }, - "shards": { - "dynamic": false, - "properties": {} - }, - "version": { - "type": "long" - }, - "queries": { - "dynamic": false, - "properties": { - "id": { - "type": "keyword" - }, - "query": { - "type": "text" - }, - "interval": { - "type": "text" - }, - "timeout": { - "type": "short" - }, - "platform": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "ecs_mapping": { - "dynamic": false, - "properties": {} - } - } + "type": { + "type": "keyword" } } }, - "osquery-pack-asset": { + "links": { "dynamic": false, "properties": { "description": { "type": "text" }, - "name": { - "type": "text" - }, - "version": { - "type": "long" - }, - "shards": { + "links": { "dynamic": false, "properties": {} }, - "queries": { - "dynamic": false, - "properties": { - "id": { - "type": "keyword" - }, - "query": { - "type": "text" - }, - "interval": { - "type": "text" - }, - "timeout": { - "type": "short" - }, - "platform": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "ecs_mapping": { - "dynamic": false, - "properties": {} - } - } + "title": { + "type": "text" } } }, - "csp-rule-template": { + "maintenance-window": { "dynamic": false, "properties": { - "metadata": { - "type": "object", - "properties": { - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "id": { - "type": "keyword" - }, - "section": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "version": { - "type": "keyword" - }, - "benchmark": { - "type": "object", - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "posture_type": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "rule_number": { - "type": "keyword" - } - } - } - } + "enabled": { + "type": "boolean" + }, + "events": { + "format": "epoch_millis||strict_date_optional_time", + "type": "date_range" } } }, - "cloud-security-posture-settings": { - "dynamic": false, - "properties": {} - }, "map": { "properties": { - "description": { - "type": "text" + "bounds": { + "dynamic": false, + "properties": {} }, - "title": { + "description": { "type": "text" }, - "version": { - "type": "integer" + "layerListJSON": { + "type": "text" }, "mapStateJSON": { "type": "text" }, - "layerListJSON": { + "title": { "type": "text" }, "uiStateJSON": { "type": "text" }, - "bounds": { - "dynamic": false, - "properties": {} + "version": { + "type": "integer" } } }, + "metrics-data-source": { + "dynamic": false, + "properties": {} + }, + "metrics-explorer-view": { + "dynamic": false, + "properties": {} + }, "ml-job": { "properties": { - "job_id": { - "type": "text", + "datafeed_id": { "fields": { "keyword": { "type": "keyword" } - } + }, + "type": "text" }, - "datafeed_id": { - "type": "text", + "job_id": { "fields": { "keyword": { "type": "keyword" } - } + }, + "type": "text" }, "type": { "type": "keyword" } } }, - "ml-trained-model": { - "properties": { - "model_id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "job": { - "properties": { - "job_id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "create_time": { - "type": "date" - } - } - } - } - }, "ml-module": { "dynamic": false, "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } + "datafeeds": { + "type": "object" }, - "title": { - "type": "text", + "defaultIndexPattern": { "fields": { "keyword": { "type": "keyword" } - } + }, + "type": "text" }, "description": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "type": { - "type": "text", "fields": { "keyword": { "type": "keyword" } - } - }, - "logo": { - "type": "object" + }, + "type": "text" }, - "defaultIndexPattern": { - "type": "text", + "id": { "fields": { "keyword": { "type": "keyword" } - } + }, + "type": "text" }, - "query": { + "jobs": { "type": "object" }, - "jobs": { + "logo": { "type": "object" }, - "datafeeds": { + "query": { "type": "object" }, "tags": { - "type": "text", "fields": { "keyword": { "type": "keyword" } - } - } - } - }, - "infrastructure-monitoring-log-view": { - "dynamic": false, - "properties": { - "name": { - "type": "text" - } - } - }, - "enterprise_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "app_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "workplace_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "slo": { - "dynamic": false, - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "text" - }, - "description": { + }, "type": "text" }, - "indicator": { - "properties": { - "type": { - "type": "keyword" - }, - "params": { - "type": "flattened" - } - } - }, - "budgetingMethod": { - "type": "keyword" - }, - "enabled": { - "type": "boolean" - }, - "tags": { - "type": "keyword" - }, - "version": { - "type": "long" - } - } - }, - "threshold-explorer-view": { - "dynamic": false, - "properties": {} - }, - "uptime-dynamic-settings": { - "dynamic": false, - "properties": {} - }, - "synthetics-privates-locations": { - "dynamic": false, - "properties": {} - }, - "synthetics-monitor": { - "dynamic": false, - "properties": { - "name": { - "type": "text", + "title": { "fields": { "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "lowercase" + "type": "keyword" } - } + }, + "type": "text" }, "type": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "urls": { - "type": "text", "fields": { "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "hosts": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "journey_id": { - "type": "keyword" - }, - "project_id": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "origin": { - "type": "keyword" - }, - "hash": { - "type": "keyword" - }, - "locations": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 256, - "fields": { - "text": { - "type": "text" - } - } - }, - "label": { - "type": "text" - } - } - }, - "custom_heartbeat_id": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "tags": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "schedule": { - "properties": { - "number": { - "type": "integer" + "type": "keyword" } - } - }, - "enabled": { - "type": "boolean" - }, - "alert": { + }, + "type": "text" + } + } + }, + "ml-trained-model": { + "properties": { + "job": { "properties": { - "status": { - "properties": { - "enabled": { - "type": "boolean" - } - } + "create_time": { + "type": "date" }, - "tls": { - "properties": { - "enabled": { - "type": "boolean" + "job_id": { + "fields": { + "keyword": { + "type": "keyword" } - } + }, + "type": "text" } } }, - "throttling": { - "properties": { - "label": { + "model_id": { + "fields": { + "keyword": { "type": "keyword" } - } + }, + "type": "text" } } }, - "uptime-synthetics-api-key": { - "dynamic": false, + "monitoring-telemetry": { "properties": { - "apiKey": { - "type": "binary" + "reportedClusterUuids": { + "type": "keyword" } } }, - "synthetics-param": { - "dynamic": false, - "properties": {} - }, "observability-onboarding-state": { "properties": { - "type": { - "type": "keyword" + "progress": { + "dynamic": false, + "type": "object" }, "state": { - "type": "object", - "dynamic": false + "dynamic": false, + "type": "object" }, - "progress": { - "type": "object", - "dynamic": false + "type": { + "type": "keyword" } } }, - "infrastructure-ui-source": { - "dynamic": false, - "properties": {} - }, - "inventory-view": { - "dynamic": false, - "properties": {} - }, - "metrics-explorer-view": { - "dynamic": false, - "properties": {} - }, - "upgrade-assistant-reindex-operation": { - "dynamic": false, + "osquery-manager-usage-metric": { "properties": { - "indexName": { - "type": "keyword" + "count": { + "type": "long" }, - "status": { - "type": "integer" + "errors": { + "type": "long" } } }, - "upgrade-assistant-ml-upgrade-operation": { - "dynamic": false, + "osquery-pack": { "properties": { - "snapshotId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "enabled": { + "type": "boolean" + }, + "name": { + "type": "text" + }, + "queries": { + "dynamic": false, + "properties": { + "ecs_mapping": { + "dynamic": false, + "properties": {} + }, + "id": { + "type": "keyword" + }, + "interval": { + "type": "text" + }, + "platform": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "timeout": { + "type": "short" + }, + "version": { + "type": "keyword" } } - } - } - }, - "monitoring-telemetry": { - "properties": { - "reportedClusterUuids": { + }, + "shards": { + "dynamic": false, + "properties": {} + }, + "updated_at": { + "type": "date" + }, + "updated_by": { "type": "keyword" + }, + "version": { + "type": "long" } } }, - "apm-telemetry": { + "osquery-pack-asset": { "dynamic": false, - "properties": {} - }, - "apm-server-schema": { "properties": { - "schemaJson": { - "type": "text", - "index": false + "description": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queries": { + "dynamic": false, + "properties": { + "ecs_mapping": { + "dynamic": false, + "properties": {} + }, + "id": { + "type": "keyword" + }, + "interval": { + "type": "text" + }, + "platform": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "timeout": { + "type": "short" + }, + "version": { + "type": "keyword" + } + } + }, + "shards": { + "dynamic": false, + "properties": {} + }, + "version": { + "type": "long" } } }, - "apm-service-group": { + "osquery-saved-query": { + "dynamic": false, "properties": { - "groupName": { - "type": "keyword" + "created_at": { + "type": "date" }, - "kuery": { + "created_by": { "type": "text" }, "description": { "type": "text" }, - "color": { + "ecs_mapping": { + "dynamic": false, + "properties": {} + }, + "id": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "timeout": { + "type": "short" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { "type": "text" + }, + "version": { + "type": "keyword" } } }, - "apm-custom-dashboards": { + "policy-settings-protection-updates-note": { "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { + "note": { + "index": false, "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" } } }, - "elastic-ai-assistant-prompts": { + "query": { + "dynamic": false, "properties": { - "id": { - "type": "keyword" - }, - "is_default": { - "type": "boolean" - }, - "is_shared": { - "type": "boolean" + "description": { + "type": "text" }, - "is_new_conversation_default": { - "type": "boolean" + "title": { + "type": "text" }, - "name": { + "titleKeyword": { "type": "keyword" - }, - "prompt_type": { + } + } + }, + "risk-engine-configuration": { + "dynamic": false, + "properties": { + "dataViewId": { "type": "keyword" }, - "content": { - "type": "keyword" + "enabled": { + "type": "boolean" }, - "updated_at": { - "type": "keyword" + "filter": { + "dynamic": false, + "properties": {} }, - "updated_by": { + "identifierType": { "type": "keyword" }, - "created_at": { + "interval": { "type": "keyword" }, - "created_by": { - "type": "keyword" + "pageSize": { + "type": "integer" + }, + "range": { + "properties": { + "end": { + "type": "keyword" + }, + "start": { + "type": "keyword" + } + } } } }, - "elastic-ai-assistant-anonymization-fields": { + "rules-settings": { + "dynamic": false, "properties": { - "id": { - "type": "keyword" - }, - "field_id": { - "type": "keyword" + "flapping": { + "properties": {} + } + } + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" }, - "default_allow": { - "type": "boolean" + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "dynamic": false, + "properties": { + "description": { + "type": "text" }, - "default_allow_replacement": { - "type": "boolean" + "title": { + "type": "text" + } + } + }, + "search-session": { + "dynamic": false, + "properties": { + "created": { + "type": "date" }, - "updated_at": { + "realmName": { "type": "keyword" }, - "updated_by": { + "realmType": { "type": "keyword" }, - "created_at": { + "sessionId": { "type": "keyword" }, - "created_by": { + "username": { "type": "keyword" } } }, - "siem-ui-timeline-note": { + "search-telemetry": { + "dynamic": false, + "properties": {} + }, + "security-rule": { + "dynamic": false, "properties": { - "eventId": { + "rule_id": { "type": "keyword" }, - "note": { - "type": "text" - }, - "created": { - "type": "date" - }, - "createdBy": { - "type": "text" - }, - "updated": { - "type": "date" - }, - "updatedBy": { - "type": "text" + "version": { + "type": "long" } } }, - "siem-ui-timeline-pinned-event": { + "security-solution-signals-migration": { + "dynamic": false, "properties": { - "eventId": { + "sourceIndex": { "type": "keyword" }, - "created": { - "type": "date" - }, - "createdBy": { - "type": "text" - }, "updated": { "type": "date" }, - "updatedBy": { - "type": "text" + "version": { + "type": "long" } } }, "siem-detection-engine-rule-actions": { "properties": { - "alertThrottle": { - "type": "keyword" - }, - "ruleAlertId": { - "type": "keyword" - }, - "ruleThrottle": { - "type": "keyword" - }, "actions": { "properties": { "actionRef": { "type": "keyword" }, - "group": { + "action_type_id": { "type": "keyword" }, - "id": { + "group": { "type": "keyword" }, - "action_type_id": { + "id": { "type": "keyword" }, "params": { @@ -2914,17 +2542,15 @@ "properties": {} } } - } - } - }, - "security-rule": { - "dynamic": false, - "properties": { - "rule_id": { + }, + "alertThrottle": { "type": "keyword" }, - "version": { - "type": "long" + "ruleAlertId": { + "type": "keyword" + }, + "ruleThrottle": { + "type": "keyword" } } }, @@ -2947,10 +2573,10 @@ "example": { "type": "text" }, - "indexes": { + "id": { "type": "keyword" }, - "id": { + "indexes": { "type": "keyword" }, "name": { @@ -2967,85 +2593,101 @@ } } }, + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, "dataProviders": { "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "text" - }, - "enabled": { - "type": "boolean" - }, - "excluded": { - "type": "boolean" - }, - "kqlQuery": { - "type": "text" - }, - "type": { - "type": "text" - }, - "queryMatch": { - "properties": { - "field": { - "type": "text" - }, - "displayField": { - "type": "text" - }, - "value": { - "type": "text" - }, - "displayValue": { - "type": "text" - }, - "operator": { - "type": "text" - } - } - }, "and": { "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "text" - }, "enabled": { "type": "boolean" }, "excluded": { "type": "boolean" }, + "id": { + "type": "keyword" + }, "kqlQuery": { "type": "text" }, - "type": { + "name": { "type": "text" }, "queryMatch": { "properties": { - "field": { - "type": "text" - }, "displayField": { "type": "text" }, - "value": { + "displayValue": { "type": "text" }, - "displayValue": { + "field": { "type": "text" }, "operator": { "type": "text" + }, + "value": { + "type": "text" } } + }, + "type": { + "type": "text" + } + } + }, + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" } } + }, + "type": { + "type": "text" + } + } + }, + "dateRange": { + "properties": { + "end": { + "type": "date" + }, + "start": { + "type": "date" } } }, @@ -3057,17 +2699,17 @@ "eventCategoryField": { "type": "text" }, - "tiebreakerField": { - "type": "text" - }, - "timestampField": { - "type": "text" - }, "query": { "type": "text" }, "size": { "type": "text" + }, + "tiebreakerField": { + "type": "text" + }, + "timestampField": { + "type": "text" } } }, @@ -3079,22 +2721,28 @@ }, "favorite": { "properties": { - "keySearch": { - "type": "text" + "favoriteDate": { + "type": "date" }, "fullName": { "type": "text" }, - "userName": { + "keySearch": { "type": "text" }, - "favoriteDate": { - "type": "date" + "userName": { + "type": "text" } } }, "filters": { "properties": { + "exists": { + "type": "text" + }, + "match_all": { + "type": "text" + }, "meta": { "properties": { "alias": { @@ -3124,23 +2772,17 @@ "params": { "type": "text" }, + "relation": { + "type": "keyword" + }, "type": { "type": "keyword" }, "value": { "type": "text" - }, - "relation": { - "type": "keyword" } } }, - "exists": { - "type": "text" - }, - "match_all": { - "type": "text" - }, "missing": { "type": "text" }, @@ -3155,154 +2797,450 @@ } } }, - "indexNames": { + "indexNames": { + "type": "text" + }, + "kqlMode": { + "type": "keyword" + }, + "kqlQuery": { + "properties": { + "filterQuery": { + "properties": { + "kuery": { + "properties": { + "expression": { + "type": "text" + }, + "kind": { + "type": "keyword" + } + } + }, + "serializedQuery": { + "type": "text" + } + } + } + } + }, + "savedSearchId": { + "type": "text" + }, + "sort": { + "dynamic": false, + "properties": { + "columnId": { + "type": "keyword" + }, + "columnType": { + "type": "keyword" + }, + "sortDirection": { + "type": "keyword" + } + } + }, + "status": { + "type": "keyword" + }, + "templateTimelineId": { + "type": "text" + }, + "templateTimelineVersion": { + "type": "integer" + }, + "timelineType": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-note": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "note": { + "type": "text" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-pinned-event": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "slo": { + "dynamic": false, + "properties": { + "budgetingMethod": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "indicator": { + "properties": { + "params": { + "type": "flattened" + }, + "type": { + "type": "keyword" + } + } + }, + "name": { + "type": "text" + }, + "tags": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "space": { + "dynamic": false, + "properties": { + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "spaces-usage-stats": { + "dynamic": false, + "properties": {} + }, + "synthetics-monitor": { + "dynamic": false, + "properties": { + "alert": { + "properties": { + "status": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "tls": { + "properties": { + "enabled": { + "type": "boolean" + } + } + } + } + }, + "custom_heartbeat_id": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "hash": { + "type": "keyword" + }, + "hosts": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "id": { + "type": "keyword" + }, + "journey_id": { + "type": "keyword" + }, + "locations": { + "properties": { + "id": { + "fields": { + "text": { + "type": "text" + } + }, + "ignore_above": 256, + "type": "keyword" + }, + "label": { + "type": "text" + } + } + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 256, + "normalizer": "lowercase", + "type": "keyword" + } + }, "type": "text" }, - "kqlMode": { + "origin": { "type": "keyword" }, - "kqlQuery": { + "project_id": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "schedule": { "properties": { - "filterQuery": { - "properties": { - "kuery": { - "properties": { - "kind": { - "type": "keyword" - }, - "expression": { - "type": "text" - } - } - }, - "serializedQuery": { - "type": "text" - } - } + "number": { + "type": "integer" } } }, - "title": { - "type": "text" - }, - "templateTimelineId": { - "type": "text" - }, - "templateTimelineVersion": { - "type": "integer" - }, - "timelineType": { + "tags": { + "fields": { + "text": { + "type": "text" + } + }, "type": "keyword" }, - "dateRange": { + "throttling": { "properties": { - "start": { - "type": "date" - }, - "end": { - "type": "date" + "label": { + "type": "keyword" } } }, - "sort": { - "dynamic": false, - "properties": { - "columnId": { - "type": "keyword" - }, - "columnType": { + "type": { + "fields": { + "keyword": { + "ignore_above": 256, "type": "keyword" - }, - "sortDirection": { + } + }, + "type": "text" + }, + "urls": { + "fields": { + "keyword": { + "ignore_above": 256, "type": "keyword" } - } + }, + "type": "text" + } + } + }, + "synthetics-param": { + "dynamic": false, + "properties": {} + }, + "synthetics-privates-locations": { + "dynamic": false, + "properties": {} + }, + "tag": { + "properties": { + "color": { + "type": "text" }, - "status": { + "description": { + "type": "text" + }, + "name": { + "type": "text" + } + } + }, + "task": { + "dynamic": false, + "properties": { + "attempts": { + "type": "integer" + }, + "enabled": { + "type": "boolean" + }, + "ownerId": { "type": "keyword" }, - "created": { + "retryAt": { "type": "date" }, - "createdBy": { - "type": "text" + "runAt": { + "type": "date" }, - "updated": { + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledAt": { "type": "date" }, - "updatedBy": { - "type": "text" + "scope": { + "type": "keyword" }, - "savedSearchId": { + "status": { + "type": "keyword" + }, + "taskType": { + "type": "keyword" + } + } + }, + "telemetry": { + "dynamic": false, + "properties": {} + }, + "threshold-explorer-view": { + "dynamic": false, + "properties": {} + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "upgrade-assistant-ml-upgrade-operation": { + "dynamic": false, + "properties": { + "snapshotId": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, "type": "text" } } }, - "endpoint:user-artifact-manifest": { + "upgrade-assistant-reindex-operation": { "dynamic": false, "properties": { - "schemaVersion": { + "indexName": { "type": "keyword" }, - "artifacts": { - "type": "nested" + "status": { + "type": "integer" } } }, - "security-solution-signals-migration": { + "uptime-dynamic-settings": { + "dynamic": false, + "properties": {} + }, + "uptime-synthetics-api-key": { "dynamic": false, "properties": { - "sourceIndex": { - "type": "keyword" + "apiKey": { + "type": "binary" + } + } + }, + "url": { + "dynamic": false, + "properties": { + "accessDate": { + "type": "date" }, - "updated": { + "createDate": { "type": "date" }, - "version": { - "type": "long" + "slug": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" } } }, - "risk-engine-configuration": { + "usage-counters": { "dynamic": false, "properties": { - "dataViewId": { + "domainId": { "type": "keyword" + } + } + }, + "visualization": { + "dynamic": false, + "properties": { + "description": { + "type": "text" }, - "enabled": { - "type": "boolean" - }, - "filter": { - "dynamic": false, + "kibanaSavedObjectMeta": { "properties": {} }, - "identifierType": { - "type": "keyword" - }, - "interval": { - "type": "keyword" + "title": { + "type": "text" }, - "pageSize": { + "version": { "type": "integer" - }, - "range": { - "properties": { - "start": { - "type": "keyword" - }, - "end": { - "type": "keyword" - } - } } } }, - "policy-settings-protection-updates-note": { - "properties": { - "note": { - "type": "text", - "index": false - } - } + "workplace_search_telemetry": { + "dynamic": false, + "properties": {} } -} +} \ No newline at end of file diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 68b31a43b3ba2..e99ca235bfad4 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -83,8 +83,6 @@ describe('checking migration metadata changes on all registered SO types', () => "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "c151324d5f85178169395eecb12bac6b96064654", "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", - "elastic-ai-assistant-anonymization-fields": "bf60d632de45cbeb69ab6b5f5579db608e67b97c", - "elastic-ai-assistant-prompts": "713a9d7e8f26b32ebb5c4042193ae29ba4059dd7", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index ba87dc7086c74..99e2692523f6d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -44,8 +44,6 @@ const previouslyRegisteredTypes = [ 'csp-rule-template', 'csp_rule', 'dashboard', - 'elastic-ai-assistant-anonymization-fields', - 'elastic-ai-assistant-prompts', 'event-annotation-group', 'endpoint:user-artifact', 'endpoint:user-artifact-manifest', diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 643dedb2bda0a..9f9c3a3c7bd58 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -203,8 +203,6 @@ describe('split .kibana index into multiple system indices', () => { "connector_token", "core-usage-stats", "csp-rule-template", - "elastic-ai-assistant-anonymization-fields", - "elastic-ai-assistant-prompts", "endpoint:user-artifact-manifest", "enterprise_search_telemetry", "epm-packages", diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index 183197850e53b..dba89d807961b 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -18,7 +18,7 @@ export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION = `${ELASTIC_AI_ export const ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND = `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL = `${ELASTIC_AI_ASSISTANT_URL}/prompts`; -export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/{id}`; +export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_bulk_action`; export const ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND = `${ELASTIC_AI_ASSISTANT_PROMPTS_URL}/_find`; export const ELASTIC_AI_ASSISTANT_ANONYMIZATION_FIELDS_URL = `${ELASTIC_AI_ASSISTANT_URL}/anonymization_fields`; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts index 5cb2b3ce071e3..f9f7900f62f24 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen.ts @@ -16,6 +16,8 @@ import { z } from 'zod'; * version: 2023-10-31 */ +import { UUID, NonEmptyString, User } from '../conversations/common_attributes.gen'; + export type BulkActionSkipReason = z.infer; export const BulkActionSkipReason = z.literal('ANONYMIZATION_FIELD_NOT_MODIFIED'); @@ -42,14 +44,20 @@ export const NormalizedAnonymizationFieldError = z.object({ export type AnonymizationFieldResponse = z.infer; export const AnonymizationFieldResponse = z.object({ - id: z.string(), - fieldId: z.string(), + id: UUID, + timestamp: NonEmptyString.optional(), + field: z.string(), defaultAllow: z.boolean().optional(), defaultAllowReplacement: z.boolean().optional(), updatedAt: z.string().optional(), updatedBy: z.string().optional(), createdAt: z.string().optional(), createdBy: z.string().optional(), + users: z.array(User).optional(), + /** + * Kibana space + */ + namespace: z.string().optional(), }); export type BulkCrudActionResults = z.infer; @@ -95,7 +103,7 @@ export const BulkActionBase = z.object({ export type AnonymizationFieldCreateProps = z.infer; export const AnonymizationFieldCreateProps = z.object({ - fieldId: z.string(), + field: z.string(), defaultAllow: z.boolean().optional(), defaultAllowReplacement: z.boolean().optional(), }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml index 0cf4a2746d33b..5df94fb538ace 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.schema.yaml @@ -100,11 +100,13 @@ components: type: object required: - id - - fieldId + - field properties: id: - type: string - fieldId: + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/UUID' + 'timestamp': + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/NonEmptyString' + field: type: string defaultAllow: type: boolean @@ -118,6 +120,13 @@ components: type: string createdBy: type: string + users: + type: array + items: + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/User' + namespace: + type: string + description: Kibana space BulkCrudActionResults: type: object @@ -207,9 +216,9 @@ components: AnonymizationFieldCreateProps: type: object required: - - fieldId + - field properties: - fieldId: + field: type: string defaultAllow: type: boolean diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts index 5b3ba908cb3d4..4121e90465edf 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts @@ -166,7 +166,7 @@ export const ConversationResponse = z.object({ */ createdAt: z.string(), replacements: Replacement.optional(), - user: User, + users: z.array(User), /** * The conversation messages. */ diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml index 1dd58fbf941c6..3fd470fc9e78e 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml @@ -134,7 +134,7 @@ components: - id - title - createdAt - - user + - users - namespace - apiConfig properties: @@ -155,8 +155,10 @@ components: type: string replacements: $ref: '#/components/schemas/Replacement' - user: - $ref: '#/components/schemas/User' + users: + type: array + items: + $ref: '#/components/schemas/User' messages: type: array items: diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts index e898e87ab3ff8..2a77b795de95f 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts @@ -36,6 +36,3 @@ export * from './actions_connector/post_actions_connector_execute_route.gen'; // KB Schemas export * from './knowledge_base/crud_kb_route.gen'; - -// Prompts Schemas -export * from './prompts/crud_prompts_route.gen'; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen.ts new file mode 100644 index 0000000000000..aa3b990ea702f --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Bulk Actions API endpoint + * version: 2023-10-31 + */ + +import { UUID, NonEmptyString, User } from '../conversations/common_attributes.gen'; + +export type BulkActionSkipReason = z.infer; +export const BulkActionSkipReason = z.literal('PROMPT_FIELD_NOT_MODIFIED'); + +export type BulkActionSkipResult = z.infer; +export const BulkActionSkipResult = z.object({ + id: z.string(), + name: z.string().optional(), + skip_reason: BulkActionSkipReason, +}); + +export type PromptDetailsInError = z.infer; +export const PromptDetailsInError = z.object({ + id: z.string(), + name: z.string().optional(), +}); + +export type NormalizedPromptError = z.infer; +export const NormalizedPromptError = z.object({ + message: z.string(), + status_code: z.number().int(), + err_code: z.string().optional(), + prompts: z.array(PromptDetailsInError), +}); + +export type PromptResponse = z.infer; +export const PromptResponse = z.object({ + id: UUID, + timestamp: NonEmptyString.optional(), + name: z.string(), + promptType: z.string(), + content: z.string(), + isNewConversationDefault: z.boolean().optional(), + isDefault: z.boolean().optional(), + isShared: z.boolean().optional(), + updatedAt: z.string().optional(), + updatedBy: z.string().optional(), + createdAt: z.string().optional(), + createdBy: z.string().optional(), + users: z.array(User).optional(), + /** + * Kibana space + */ + namespace: z.string().optional(), +}); + +export type BulkCrudActionResults = z.infer; +export const BulkCrudActionResults = z.object({ + updated: z.array(PromptResponse), + created: z.array(PromptResponse), + deleted: z.array(z.string()), + skipped: z.array(BulkActionSkipResult), +}); + +export type BulkCrudActionSummary = z.infer; +export const BulkCrudActionSummary = z.object({ + failed: z.number().int(), + skipped: z.number().int(), + succeeded: z.number().int(), + total: z.number().int(), +}); + +export type BulkCrudActionResponse = z.infer; +export const BulkCrudActionResponse = z.object({ + success: z.boolean().optional(), + status_code: z.number().int().optional(), + message: z.string().optional(), + prompts_count: z.number().int().optional(), + attributes: z.object({ + results: BulkCrudActionResults, + summary: BulkCrudActionSummary, + errors: z.array(NormalizedPromptError).optional(), + }), +}); + +export type BulkActionBase = z.infer; +export const BulkActionBase = z.object({ + /** + * Query to filter promps + */ + query: z.string().optional(), + /** + * Array of prompts IDs + */ + ids: z.array(z.string()).min(1).optional(), +}); + +export type PromptCreateProps = z.infer; +export const PromptCreateProps = z.object({ + name: z.string(), + promptType: z.string().optional(), + content: z.string(), + isNewConversationDefault: z.boolean().optional(), + isDefault: z.boolean().optional(), + isShared: z.boolean().optional(), +}); + +export type PromptUpdateProps = z.infer; +export const PromptUpdateProps = z.object({ + id: z.string(), + content: z.string().optional(), + isNewConversationDefault: z.boolean().optional(), + isDefault: z.boolean().optional(), + isShared: z.boolean().optional(), +}); + +export type PerformBulkActionRequestBody = z.infer; +export const PerformBulkActionRequestBody = z.object({ + delete: BulkActionBase.optional(), + create: z.array(PromptCreateProps).optional(), + update: z.array(PromptUpdateProps).optional(), +}); +export type PerformBulkActionRequestBodyInput = z.input; + +export type PerformBulkActionResponse = z.infer; +export const PerformBulkActionResponse = BulkCrudActionResponse; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.schema.yaml new file mode 100644 index 0000000000000..d43d5144ce550 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.schema.yaml @@ -0,0 +1,259 @@ +openapi: 3.0.0 +info: + title: Bulk Actions API endpoint + version: '2023-10-31' +paths: + /api/elastic_assistant/prompts/_bulk_action: + post: + operationId: PerformBulkAction + x-codegen-enabled: true + summary: Applies a bulk action to multiple prompts + description: The bulk action is applied to all prompts that match the filter or to the list of prompts by their IDs. + tags: + - Bulk API + requestBody: + content: + application/json: + schema: + type: object + properties: + delete: + $ref: '#/components/schemas/BulkActionBase' + create: + type: array + items: + $ref: '#/components/schemas/PromptCreateProps' + update: + type: array + items: + $ref: '#/components/schemas/PromptUpdateProps' + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: '#/components/schemas/BulkCrudActionResponse' + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string + +components: + schemas: + BulkActionSkipReason: + type: string + enum: + - PROMPT_FIELD_NOT_MODIFIED + + BulkActionSkipResult: + type: object + properties: + id: + type: string + name: + type: string + skip_reason: + $ref: '#/components/schemas/BulkActionSkipReason' + required: + - id + - skip_reason + + PromptDetailsInError: + type: object + properties: + id: + type: string + name: + type: string + required: + - id + + NormalizedPromptError: + type: object + properties: + message: + type: string + status_code: + type: integer + err_code: + type: string + prompts: + type: array + items: + $ref: '#/components/schemas/PromptDetailsInError' + required: + - message + - status_code + - prompts + + PromptResponse: + type: object + required: + - id + - name + - promptType + - content + properties: + id: + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/UUID' + 'timestamp': + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/NonEmptyString' + name: + type: string + promptType: + type: string + content: + type: string + isNewConversationDefault: + type: boolean + isDefault: + type: boolean + isShared: + type: boolean + updatedAt: + type: string + updatedBy: + type: string + createdAt: + type: string + createdBy: + type: string + users: + type: array + items: + $ref: '../conversations/common_attributes.schema.yaml#/components/schemas/User' + namespace: + type: string + description: Kibana space + + BulkCrudActionResults: + type: object + properties: + updated: + type: array + items: + $ref: '#/components/schemas/PromptResponse' + created: + type: array + items: + $ref: '#/components/schemas/PromptResponse' + deleted: + type: array + items: + type: string + skipped: + type: array + items: + $ref: '#/components/schemas/BulkActionSkipResult' + required: + - updated + - created + - deleted + - skipped + + BulkCrudActionSummary: + type: object + properties: + failed: + type: integer + skipped: + type: integer + succeeded: + type: integer + total: + type: integer + required: + - failed + - skipped + - succeeded + - total + + BulkCrudActionResponse: + type: object + properties: + success: + type: boolean + status_code: + type: integer + message: + type: string + prompts_count: + type: integer + attributes: + type: object + properties: + results: + $ref: '#/components/schemas/BulkCrudActionResults' + summary: + $ref: '#/components/schemas/BulkCrudActionSummary' + errors: + type: array + items: + $ref: '#/components/schemas/NormalizedPromptError' + required: + - results + - summary + required: + - attributes + + + BulkActionBase: + x-inline: true + type: object + properties: + query: + type: string + description: Query to filter promps + ids: + type: array + description: Array of prompts IDs + minItems: 1 + items: + type: string + + PromptCreateProps: + type: object + required: + - name + - content + + properties: + name: + type: string + promptType: + type: string + content: + type: string + isNewConversationDefault: + type: boolean + isDefault: + type: boolean + isShared: + type: boolean + + PromptUpdateProps: + type: object + required: + - id + properties: + id: + type: string + content: + type: string + isNewConversationDefault: + type: boolean + isDefault: + type: boolean + isShared: + type: boolean + \ No newline at end of file diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts deleted file mode 100644 index d2edd0384e598..0000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen.ts +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from 'zod'; - -/* - * NOTICE: Do not edit this file manually. - * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. - * - * info: - * title: Create Prompt API endpoint - * version: 2023-10-31 - */ - -/** - * AI assistant create prompt settings. - */ -export type PromptCreateProps = z.infer; -export const PromptCreateProps = z.object({ - /** - * Prompt content. - */ - content: z.string(), - /** - * Prompt name. - */ - name: z.string(), - /** - * Prompt type. - */ - promptType: z.string(), - /** - * Is default prompt. - */ - isDefault: z.boolean().optional(), - /** - * Is shared prompt. - */ - isShared: z.boolean().optional(), - /** - * Is default prompt. - */ - isNewConversationDefault: z.boolean().optional(), -}); - -/** - * AI assistant update prompt settings. - */ -export type PromptUpdateProps = z.infer; -export const PromptUpdateProps = z.object({ - /** - * Prompt content. - */ - content: z.string().optional(), - /** - * Prompt name. - */ - name: z.string().optional(), - /** - * Prompt type. - */ - promptType: z.string().optional(), - /** - * Is shared prompt. - */ - isShared: z.boolean().optional(), - /** - * Is default prompt. - */ - isNewConversationDefault: z.boolean().optional(), -}); - -/** - * AI assistant prompt. - */ -export type PromptResponse = z.infer; -export const PromptResponse = z.object({ - id: z.string(), - /** - * Prompt content. - */ - content: z.string(), - /** - * Prompt name. - */ - name: z.string().optional(), - /** - * Prompt type. - */ - promptType: z.string().optional(), - /** - * Is default prompt. - */ - isDefault: z.boolean().optional(), - /** - * Is shared prompt. - */ - isShared: z.boolean().optional(), - /** - * Is default prompt. - */ - isNewConversationDefault: z.boolean().optional(), - /** - * The last time prompt was updated. - */ - updatedAt: z.string().optional(), - /** - * The last time prompt was updated. - */ - createdAt: z.string().optional(), - /** - * User who was updated prompt. - */ - updatedBy: z.string().optional(), - /** - * User who was created prompt. - */ - createdBy: z.string().optional(), -}); - -export type CreatePromptRequestBody = z.infer; -export const CreatePromptRequestBody = PromptCreateProps; -export type CreatePromptRequestBodyInput = z.input; - -export type CreatePromptResponse = z.infer; -export const CreatePromptResponse = PromptResponse; - -export type DeletePromptRequestParams = z.infer; -export const DeletePromptRequestParams = z.object({ - /** - * The prompt's `id` value. - */ - id: z.string(), -}); -export type DeletePromptRequestParamsInput = z.input; - -export type DeletePromptResponse = z.infer; -export const DeletePromptResponse = PromptResponse; - -export type ReadPromptRequestParams = z.infer; -export const ReadPromptRequestParams = z.object({ - /** - * The prompt's `id` value. - */ - id: z.string(), -}); -export type ReadPromptRequestParamsInput = z.input; - -export type ReadPromptResponse = z.infer; -export const ReadPromptResponse = PromptResponse; - -export type UpdatePromptRequestParams = z.infer; -export const UpdatePromptRequestParams = z.object({ - /** - * The prompt's `id` value. - */ - id: z.string(), -}); -export type UpdatePromptRequestParamsInput = z.input; - -export type UpdatePromptRequestBody = z.infer; -export const UpdatePromptRequestBody = PromptUpdateProps; -export type UpdatePromptRequestBodyInput = z.input; - -export type UpdatePromptResponse = z.infer; -export const UpdatePromptResponse = PromptResponse; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml deleted file mode 100644 index 39736d61a3786..0000000000000 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.schema.yaml +++ /dev/null @@ -1,240 +0,0 @@ -openapi: 3.0.0 -info: - title: Create Prompt API endpoint - version: '2023-10-31' -paths: - /api/elastic_assistant/prompts: - post: - operationId: CreatePrompt - x-codegen-enabled: true - description: Create a prompt - summary: Create a prompt - tags: - - Prompt API - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/PromptCreateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: '#/components/schemas/PromptResponse' - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string - - /api/elastic_assistant/prompts/{id}: - get: - operationId: ReadPrompt - x-codegen-enabled: true - description: Read a single prompt - summary: Read a single prompt - tags: - - Prompts API - parameters: - - name: id - in: path - required: true - description: The prompt's `id` value. - schema: - type: string - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: '#/components/schemas/PromptResponse' - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string - put: - operationId: UpdatePrompt - x-codegen-enabled: true - description: Update a single prompt - summary: Update a single prompt - tags: - - Prompt API - parameters: - - name: id - in: path - required: true - description: The prompt's `id` value. - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/PromptUpdateProps' - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: '#/components/schemas/PromptResponse' - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string - delete: - operationId: DeletePrompt - x-codegen-enabled: true - description: Deletes a single prompt using the `id` field. - summary: Deletes a single prompt - tags: - - Prompt API - parameters: - - name: id - in: path - required: true - description: The prompt's `id` value. - schema: - type: string - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: '#/components/schemas/PromptResponse' - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string - -components: - schemas: - PromptCreateProps: - type: object - description: AI assistant create prompt settings. - required: - - 'content' - - 'name' - - 'promptType' - properties: - content: - type: string - description: Prompt content. - name: - type: string - description: Prompt name. - promptType: - type: string - description: Prompt type. - isDefault: - description: Is default prompt. - type: boolean - isShared: - description: Is shared prompt. - type: boolean - isNewConversationDefault: - description: Is default prompt. - type: boolean - - PromptUpdateProps: - type: object - description: AI assistant update prompt settings. - properties: - content: - type: string - description: Prompt content. - name: - type: string - description: Prompt name. - promptType: - type: string - description: Prompt type. - isShared: - description: Is shared prompt. - type: boolean - isNewConversationDefault: - description: Is default prompt. - type: boolean - - PromptResponse: - type: object - description: AI assistant prompt. - required: - - 'timestamp' - - 'content' - - 'role' - - 'id' - properties: - id: - type: string - content: - type: string - description: Prompt content. - name: - type: string - description: Prompt name. - promptType: - type: string - description: Prompt type. - isDefault: - description: Is default prompt. - type: boolean - isShared: - description: Is shared prompt. - type: boolean - isNewConversationDefault: - description: Is default prompt. - type: boolean - updatedAt: - description: The last time prompt was updated. - type: string - createdAt: - description: The last time prompt was updated. - type: string - updatedBy: - description: User who was updated prompt. - type: string - createdBy: - description: User who was created prompt. - type: string diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts index 537e9ded995d0..7400b11f25c7a 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen.ts @@ -17,10 +17,10 @@ import { ArrayFromString } from '@kbn/zod-helpers'; * version: 2023-10-31 */ -import { PromptResponse } from './crud_prompts_route.gen'; +import { PromptResponse } from './bulk_crud_prompts_route.gen'; export type FindPromptsSortField = z.infer; -export const FindPromptsSortField = z.enum(['created_at', 'is_default', 'title', 'updated_at']); +export const FindPromptsSortField = z.enum(['created_at', 'is_default', 'name', 'updated_at']); export type FindPromptsSortFieldEnum = typeof FindPromptsSortField.enum; export const FindPromptsSortFieldEnum = FindPromptsSortField.enum; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml index fb197c8b42476..b5d3b25ca2018 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/prompts/find_prompts_route.schema.yaml @@ -71,7 +71,7 @@ paths: data: type: array items: - $ref: './crud_prompts_route.schema.yaml#/components/schemas/PromptResponse' + $ref: './bulk_crud_prompts_route.schema.yaml#/components/schemas/PromptResponse' required: - page - perPage @@ -98,7 +98,7 @@ components: enum: - 'created_at' - 'is_default' - - 'title' + - 'name' - 'updated_at' SortOrder: diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index 7fb45bd40f30e..2aa2f09900fe3 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -146,7 +146,7 @@ describe('AI Assistant Service', () => { }); }); - describe('createAIAssistantDatastreamClient()', () => { + describe('createAIAssistantConversationsDataClient()', () => { let assistantService: AIAssistantService; beforeEach(async () => { (AIAssistantConversationsDataClient as jest.Mock).mockImplementation( @@ -168,7 +168,7 @@ describe('AI Assistant Service', () => { async () => assistantService.isInitialized() === true ); - await assistantService.createAIAssistantDatastreamClient({ + await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -204,7 +204,7 @@ describe('AI Assistant Service', () => { expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); expect(clusterClient.indices.create).not.toHaveBeenCalled(); - const result = await assistantService.createAIAssistantDatastreamClient({ + const result = await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -263,12 +263,12 @@ describe('AI Assistant Service', () => { // call createAIAssistantConversationsDataClient at the same time which will trigger the retries const result = await Promise.all([ - assistantService.createAIAssistantDatastreamClient({ + assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, }), - assistantService.createAIAssistantDatastreamClient({ + assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -334,7 +334,7 @@ describe('AI Assistant Service', () => { async () => assistantService.isInitialized() === true ); - const result = await assistantService.createAIAssistantDatastreamClient({ + const result = await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -400,7 +400,7 @@ describe('AI Assistant Service', () => { await new Promise((r) => setTimeout(r, delayMs)); } - return assistantService.createAIAssistantDatastreamClient({ + return assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -478,7 +478,7 @@ describe('AI Assistant Service', () => { await new Promise((r) => setTimeout(r, delayMs)); } - return assistantService.createAIAssistantDatastreamClient({ + return assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, @@ -525,7 +525,7 @@ describe('AI Assistant Service', () => { expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); expect(clusterClient.indices.create).not.toHaveBeenCalled(); - const result = await assistantService.createAIAssistantDatastreamClient({ + const result = await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'test', currentUser: mockUser1, @@ -578,7 +578,7 @@ describe('AI Assistant Service', () => { expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); expect(clusterClient.indices.create).not.toHaveBeenCalled(); - const result = await assistantService.createAIAssistantDatastreamClient({ + const result = await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'test', currentUser: mockUser1, @@ -633,7 +633,7 @@ describe('AI Assistant Service', () => { async () => assistantService.isInitialized() === true ); - const result = await assistantService.createAIAssistantDatastreamClient({ + const result = await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'test', currentUser: mockUser1, @@ -780,7 +780,7 @@ describe('AI Assistant Service', () => { async () => assistantService.isInitialized() === true ); - await assistantService.createAIAssistantDatastreamClient({ + await assistantService.createAIAssistantConversationsDataClient({ logger, spaceId: 'default', currentUser: mockUser1, diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 80487472b1e39..89a3d272d072d 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; +import { DataStreamSpacesAdapter, FieldMap } from '@kbn/data-stream-adapter'; import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; @@ -13,6 +13,8 @@ import { AuthenticatedUser } from '@kbn/security-plugin/server'; import { Subject } from 'rxjs'; import { AssistantResourceNames } from '../types'; import { AIAssistantConversationsDataClient } from '../conversations_data_client'; +import { AIAssistantPromtsDataClient } from '../promts_data_client'; +import { AIAssistantAnonymizationFieldsDataClient } from '../anonymization_fields_data_client'; import { InitializationPromise, ResourceInstallationHelper, @@ -20,7 +22,9 @@ import { errorResult, successResult, } from './create_resource_installation_helper'; -import { conversationsFieldMap } from './conversation_configuration_type'; +import { conversationsFieldMap } from '../conversations_data_client/conversations_configuration_type'; +import { assistantPromptsFieldMap } from '../promts_data_client/prompts_configuration_type'; +import { assistantAnonymizationFieldsFieldMap } from '../anonymization_fields_data_client/anonymization_fields_configuration_type'; const TOTAL_FIELDS_LIMIT = 2500; @@ -42,7 +46,9 @@ export interface CreateAIAssistantClientParams { currentUser: AuthenticatedUser | null; } -export type CreateConversationsDataStream = (params: { +export type CreateDataStream = (params: { + resource: 'conversations' | 'prompts' | 'anonymizationFields'; + fieldMap: FieldMap; kibanaVersion: string; spaceId?: string; }) => DataStreamSpacesAdapter; @@ -51,13 +57,27 @@ export class AIAssistantService { private initialized: boolean; private isInitializing: boolean = false; private conversationsDataStream: DataStreamSpacesAdapter; + private promptsDataStream: DataStreamSpacesAdapter; + private anonymizationFieldsDataStream: DataStreamSpacesAdapter; private resourceInitializationHelper: ResourceInstallationHelper; private initPromise: Promise; constructor(private readonly options: AIAssistantServiceOpts) { this.initialized = false; - this.conversationsDataStream = this.createConversationsDataStream({ + this.conversationsDataStream = this.createDataStream({ + resource: 'conversations', kibanaVersion: options.kibanaVersion, + fieldMap: conversationsFieldMap, + }); + this.promptsDataStream = this.createDataStream({ + resource: 'prompts', + kibanaVersion: options.kibanaVersion, + fieldMap: assistantPromptsFieldMap, + }); + this.anonymizationFieldsDataStream = this.createDataStream({ + resource: 'anonymizationFields', + kibanaVersion: options.kibanaVersion, + fieldMap: assistantAnonymizationFieldsFieldMap, }); this.initPromise = this.initializeResources(); @@ -73,26 +93,23 @@ export class AIAssistantService { return this.initialized; } - private createConversationsDataStream: CreateConversationsDataStream = ({ kibanaVersion }) => { - const conversationsDataStream = new DataStreamSpacesAdapter( - this.resourceNames.aliases.conversations, - { - kibanaVersion, - totalFieldsLimit: TOTAL_FIELDS_LIMIT, - } - ); + private createDataStream: CreateDataStream = ({ resource, kibanaVersion, fieldMap }) => { + const newDataStream = new DataStreamSpacesAdapter(this.resourceNames.aliases[resource], { + kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + }); - conversationsDataStream.setComponentTemplate({ - name: this.resourceNames.componentTemplate.conversations, - fieldMap: conversationsFieldMap, + newDataStream.setComponentTemplate({ + name: this.resourceNames.componentTemplate[resource], + fieldMap, }); - conversationsDataStream.setIndexTemplate({ - name: this.resourceNames.indexTemplate.conversations, - componentTemplateRefs: [this.resourceNames.componentTemplate.conversations], + newDataStream.setIndexTemplate({ + name: this.resourceNames.indexTemplate[resource], + componentTemplateRefs: [this.resourceNames.componentTemplate[resource]], }); - return conversationsDataStream; + return newDataStream; }; private async initializeResources(): Promise { @@ -106,6 +123,18 @@ export class AIAssistantService { logger: this.options.logger, pluginStop$: this.options.pluginStop$, }); + + await this.promptsDataStream.install({ + esClient, + logger: this.options.logger, + pluginStop$: this.options.pluginStop$, + }); + + await this.anonymizationFieldsDataStream.install({ + esClient, + logger: this.options.logger, + pluginStop$: this.options.pluginStop$, + }); } catch (error) { this.options.logger.error(`Error initializing AI assistant resources: ${error.message}`); this.initialized = false; @@ -120,18 +149,26 @@ export class AIAssistantService { private readonly resourceNames: AssistantResourceNames = { componentTemplate: { conversations: getResourceName('component-template-conversations'), + prompts: getResourceName('component-template-prompts'), + anonymizationFields: getResourceName('component-template-anonymization-fields'), kb: getResourceName('component-template-kb'), }, aliases: { conversations: getResourceName('conversations'), + prompts: getResourceName('prompts'), + anonymizationFields: getResourceName('anonymization-fields'), kb: getResourceName('kb'), }, indexPatterns: { conversations: getResourceName('conversations*'), + prompts: getResourceName('prompts*'), + anonymizationFields: getResourceName('anonymization-fields*'), kb: getResourceName('kb*'), }, indexTemplate: { conversations: getResourceName('index-template-conversations'), + prompts: getResourceName('index-template-prompts'), + anonymizationFields: getResourceName('index-template-anonymization-fields'), kb: getResourceName('index-template-kb'), }, pipelines: { @@ -139,9 +176,7 @@ export class AIAssistantService { }, }; - public async createAIAssistantDatastreamClient( - opts: CreateAIAssistantClientParams - ): Promise { + private async checkResourcesInstallation(opts: CreateAIAssistantClientParams) { // Check if resources installation has succeeded const { result: initialized, error } = await this.getSpaceResourcesInitializationPromise( opts.spaceId @@ -185,6 +220,12 @@ export class AIAssistantService { ); } } + } + + public async createAIAssistantConversationsDataClient( + opts: CreateAIAssistantClientParams + ): Promise { + await this.checkResourcesInstallation(opts); return new AIAssistantConversationsDataClient({ logger: this.options.logger, @@ -196,6 +237,36 @@ export class AIAssistantService { }); } + public async createAIAssistantPromptsDataClient( + opts: CreateAIAssistantClientParams + ): Promise { + await this.checkResourcesInstallation(opts); + + return new AIAssistantPromtsDataClient({ + logger: this.options.logger, + elasticsearchClientPromise: this.options.elasticsearchClientPromise, + spaceId: opts.spaceId, + kibanaVersion: this.options.kibanaVersion, + indexPatternsResorceName: this.resourceNames.aliases.prompts, + currentUser: opts.currentUser, + }); + } + + public async createAIAssistantAnonymizationFieldsDataClient( + opts: CreateAIAssistantClientParams + ): Promise { + await this.checkResourcesInstallation(opts); + + return new AIAssistantAnonymizationFieldsDataClient({ + logger: this.options.logger, + elasticsearchClientPromise: this.options.elasticsearchClientPromise, + spaceId: opts.spaceId, + kibanaVersion: this.options.kibanaVersion, + indexPatternsResorceName: this.resourceNames.aliases.anonymizationFields, + currentUser: opts.currentUser, + }); + } + public async getSpaceResourcesInitializationPromise( spaceId: string | undefined = DEFAULT_NAMESPACE_STRING ): Promise { @@ -218,9 +289,24 @@ export class AIAssistantService { ) { try { this.options.logger.debug(`Initializing spaceId level resources for AIAssistantService`); - let indexName = await this.conversationsDataStream.getInstalledSpaceName(spaceId); - if (!indexName) { - indexName = await this.conversationsDataStream.installSpace(spaceId); + let conversationsIndexName = await this.conversationsDataStream.getInstalledSpaceName( + spaceId + ); + if (!conversationsIndexName) { + conversationsIndexName = await this.conversationsDataStream.installSpace(spaceId); + } + + let promptsIndexName = await this.promptsDataStream.getInstalledSpaceName(spaceId); + if (!promptsIndexName) { + promptsIndexName = await this.promptsDataStream.installSpace(spaceId); + } + + let anonymizationFieldsIndexName = + await this.anonymizationFieldsDataStream.getInstalledSpaceName(spaceId); + if (!anonymizationFieldsIndexName) { + anonymizationFieldsIndexName = await this.anonymizationFieldsDataStream.installSpace( + spaceId + ); } } catch (error) { this.options.logger.error( diff --git a/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/anonymization_fields_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/anonymization_fields_configuration_type.ts new file mode 100644 index 0000000000000..c97a20a2ad521 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/anonymization_fields_configuration_type.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldMap } from '@kbn/data-stream-adapter'; + +export const assistantAnonymizationFieldsFieldMap: FieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: false, + }, + id: { + type: 'keyword', + array: false, + required: true, + }, + field: { + type: 'keyword', + array: false, + required: false, + }, + default_allow: { + type: 'boolean', + array: false, + required: false, + }, + default_allow_replacement: { + type: 'boolean', + array: false, + required: false, + }, + updated_at: { + type: 'date', + array: false, + required: false, + }, + updated_by: { + type: 'keyword', + array: false, + required: false, + }, + created_at: { + type: 'date', + array: false, + required: false, + }, + created_by: { + type: 'keyword', + array: false, + required: false, + }, + users: { + type: 'nested', + array: true, + required: false, + }, + 'users.id': { + type: 'keyword', + array: false, + required: false, + }, + 'users.name': { + type: 'keyword', + array: false, + required: false, + }, +}; diff --git a/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/find_anonymization_fields.ts b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/find_anonymization_fields.ts new file mode 100644 index 0000000000000..2cac26aacf4f6 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/find_anonymization_fields.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingRuntimeFields, Sort } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import { estypes } from '@elastic/elasticsearch'; +import { EsQueryConfig, Query, buildEsQuery } from '@kbn/es-query'; +import { FindAnonymizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; +import { SearchEsAnonymizationFieldsSchema } from './types'; +import { transformESToAnonymizationFields } from './helpers'; + +interface FindAnonymizationFieldsOptions { + filter?: string; + fields?: string[]; + perPage: number; + page: number; + sortField?: string; + sortOrder?: estypes.SortOrder; + esClient: ElasticsearchClient; + anonymizationFieldsIndex: string; + runtimeMappings?: MappingRuntimeFields | undefined; +} + +export const findAnonymizationFields = async ({ + esClient, + filter, + page, + perPage, + sortField, + anonymizationFieldsIndex, + fields, + sortOrder, +}: FindAnonymizationFieldsOptions): Promise => { + const query = getQueryFilter({ filter }); + let sort: Sort | undefined; + const ascOrDesc = sortOrder ?? ('asc' as const); + if (sortField != null) { + sort = [{ [sortField]: ascOrDesc }]; + } else { + sort = { + updated_at: { + order: 'desc', + }, + }; + } + const response = await esClient.search({ + body: { + query, + track_total_hits: true, + sort, + }, + _source: true, + from: (page - 1) * perPage, + ignore_unavailable: true, + index: anonymizationFieldsIndex, + seq_no_primary_term: true, + size: perPage, + }); + return { + data: transformESToAnonymizationFields(response), + page, + perPage, + total: + (typeof response.hits.total === 'number' + ? response.hits.total // This format is to be removed in 8.0 + : response.hits.total?.value) ?? 0, + }; +}; + +export interface GetQueryFilterOptions { + filter?: string; +} + +export const getQueryFilter = ({ filter }: GetQueryFilterOptions) => { + const kqlQuery: Query | Query[] = filter + ? { + language: 'kuery', + query: filter, + } + : []; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + dateFormatTZ: 'Zulu', + ignoreFilterIfFieldNotInIndex: false, + queryStringOptions: { analyze_wildcard: true }, + }; + + return buildEsQuery(undefined, kqlQuery, [], config); +}; diff --git a/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/helpers.ts b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/helpers.ts new file mode 100644 index 0000000000000..74a7778fb977c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/helpers.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { + AnonymizationFieldResponse, + AnonymizationFieldUpdateProps, +} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; +import { SearchEsAnonymizationFieldsSchema, UpdateAnonymizationFieldSchema } from './types'; + +export const transformESToAnonymizationFields = ( + response: estypes.SearchResponse +): AnonymizationFieldResponse[] => { + return response.hits.hits + .filter((hit) => hit._source !== undefined) + .map((hit) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const anonymizationFieldSchema = hit._source!; + const anonymizationField: AnonymizationFieldResponse = { + timestamp: anonymizationFieldSchema['@timestamp'], + createdAt: anonymizationFieldSchema.created_at, + users: + anonymizationFieldSchema.users?.map((user) => ({ + id: user.id, + name: user.name, + })) ?? [], + field: anonymizationFieldSchema.field, + defaultAllow: anonymizationFieldSchema.default_allow, + defaultAllowReplacement: anonymizationFieldSchema.default_allow_replacement, + updatedAt: anonymizationFieldSchema.updated_at, + namespace: anonymizationFieldSchema.namespace, + id: hit._id, + }; + + return anonymizationField; + }); +}; + +export const transformToUpdateScheme = ( + updatedAt: string, + { defaultAllow, defaultAllowReplacement }: AnonymizationFieldUpdateProps +): UpdateAnonymizationFieldSchema => { + return { + updated_at: updatedAt, + default_allow: defaultAllow, + default_allow_replacement: defaultAllowReplacement, + }; +}; + +export const getUpdateScript = ({ + anonymizationField, + updatedAt, + isPatch, +}: { + anonymizationField: AnonymizationFieldUpdateProps; + updatedAt: string; + isPatch?: boolean; +}) => { + return { + source: ` + if (params.assignEmpty == true || params.containsKey('default_allow')) { + ctx._source.default_allow = params.default_allow; + } + if (params.assignEmpty == true || params.containsKey('default_allow_replacement')) { + ctx._source.default_allow_replacement = params.default_allow_replacement; + } + ctx._source.updated_at = params.updated_at; + `, + lang: 'painless', + params: { + ...transformToUpdateScheme(updatedAt, anonymizationField), // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !(isPatch ?? true), + }, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/index.ts new file mode 100644 index 0000000000000..f83b9bd0183a5 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/index.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; +import { estypes } from '@elastic/elasticsearch'; +import { FindAnonymizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; +import { AnonymizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; +import { DocumentsDataWriter } from '../lib/data_client/documents_data_writer'; +import { IIndexPatternString } from '../types'; +import { getIndexTemplateAndPattern } from '../lib/data_client/helper'; +import { findAnonymizationFields } from './find_anonymization_fields'; + +export interface AIAssistantAnonymizationFieldsDataClientParams { + elasticsearchClientPromise: Promise; + kibanaVersion: string; + spaceId: string; + logger: Logger; + indexPatternsResorceName: string; + currentUser: AuthenticatedUser | null; +} + +/** + * Class for use for anonymization fields that are used for AI assistant. + */ +export class AIAssistantAnonymizationFieldsDataClient { + /** Kibana space id the anonymization fields are part of */ + private readonly spaceId: string; + + /** User creating, modifying, deleting, or updating a anonymization fields */ + private readonly currentUser: AuthenticatedUser | null; + + private writerCache: Map = new Map(); + + private indexTemplateAndPattern: IIndexPatternString; + + constructor(private readonly options: AIAssistantAnonymizationFieldsDataClientParams) { + this.indexTemplateAndPattern = getIndexTemplateAndPattern( + this.options.indexPatternsResorceName, + this.options.spaceId ?? DEFAULT_NAMESPACE_STRING + ); + this.currentUser = this.options.currentUser; + this.spaceId = this.options.spaceId; + } + + public getWriter = async (): Promise => { + const spaceId = this.spaceId; + if (this.writerCache.get(spaceId)) { + return this.writerCache.get(spaceId) as DocumentsDataWriter; + } + await this.initializeWriter(spaceId, this.indexTemplateAndPattern.alias); + return this.writerCache.get(spaceId) as DocumentsDataWriter; + }; + + private async initializeWriter(spaceId: string, index: string): Promise { + const esClient = await this.options.elasticsearchClientPromise; + const writer = new DocumentsDataWriter({ + esClient, + spaceId, + index, + logger: this.options.logger, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + }); + + this.writerCache.set(spaceId, writer); + return writer; + } + + public getReader = async (options: { spaceId?: string } = {}) => { + const indexPatterns = this.indexTemplateAndPattern.alias; + + return { + search: async < + TSearchRequest extends ESSearchRequest, + TAnonymizationFieldDoc = Partial + >( + request: TSearchRequest + ): Promise> => { + try { + const esClient = await this.options.elasticsearchClientPromise; + return (await esClient.search({ + ...request, + index: indexPatterns, + ignore_unavailable: true, + seq_no_primary_term: true, + })) as unknown as ESSearchResponse; + } catch (err) { + this.options.logger.error( + `Error performing search in AIAssistantDataClient - ${err.message}` + ); + throw err; + } + }, + }; + }; + + public findAnonymizationFields = async ({ + perPage, + page, + sortField, + sortOrder, + filter, + fields, + }: { + perPage: number; + page: number; + sortField?: string; + sortOrder?: string; + filter?: string; + fields?: string[]; + }): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + return findAnonymizationFields({ + esClient, + fields, + page, + perPage, + filter, + sortField, + anonymizationFieldsIndex: this.indexTemplateAndPattern.alias, + sortOrder: sortOrder as estypes.SortOrder, + }); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/types.ts new file mode 100644 index 0000000000000..b8ebc070edb89 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/anonymization_fields_data_client/types.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface SearchEsAnonymizationFieldsSchema { + id: string; + '@timestamp': string; + created_at: string; + created_by: string; + field: string; + default_allow_replacement?: boolean; + default_allow?: boolean; + users?: Array<{ + id?: string; + name?: string; + }>; + updated_at?: string; + updated_by?: string; + namespace: string; +} + +export interface UpdateAnonymizationFieldSchema { + '@timestamp'?: string; + default_allow_replacement?: boolean; + default_allow?: boolean; + users?: Array<{ + id?: string; + name?: string; + }>; + updated_at?: string; + updated_by?: string; +} diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index 81dc563415ac8..6401f4133d635 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -97,9 +97,7 @@ export const transformToUpdateScheme = (updatedAt: string, messages: Message[]) '@timestamp': message.timestamp, content: message.content, is_error: message.isError, - presentation: message.presentation, reader: message.reader, - replacements: message.replacements, role: message.role, trace_data: { trace_id: message.traceData?.traceId, diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts similarity index 71% rename from x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts rename to x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts index d246845ee5f1d..3a941c51fbe5f 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/conversation_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts @@ -4,9 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { FieldMap } from '@kbn/alerts-as-data-utils'; -import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; -import { IIndexPatternString } from '../types'; +import { FieldMap } from '@kbn/data-stream-adapter'; export const conversationsFieldMap: FieldMap = { '@timestamp': { @@ -14,12 +12,17 @@ export const conversationsFieldMap: FieldMap = { array: false, required: false, }, - 'user.id': { + users: { + type: 'nested', + array: true, + required: false, + }, + 'users.id': { type: 'keyword', array: false, required: true, }, - 'user.name': { + 'users.name': { type: 'keyword', array: false, required: false, @@ -84,21 +87,6 @@ export const conversationsFieldMap: FieldMap = { array: false, required: false, }, - 'messages.presentation': { - type: 'object', - array: false, - required: false, - }, - 'messages.presentation.delay': { - type: 'long', - array: false, - required: false, - }, - 'messages.presentation.stream': { - type: 'boolean', - array: false, - required: false, - }, 'messages.trace_data': { type: 'object', array: false, @@ -145,18 +133,3 @@ export const conversationsFieldMap: FieldMap = { required: false, }, } as const; - -export const getIndexTemplateAndPattern = ( - context: string, - namespace?: string -): IIndexPatternString => { - const concreteNamespace = namespace ? namespace : DEFAULT_NAMESPACE_STRING; - const pattern = `${context}`; - const patternWithNamespace = `${pattern}-${concreteNamespace}`; - return { - pattern: `${patternWithNamespace}*`, - basePattern: `${pattern}-*`, - name: `${patternWithNamespace}-000001`, - alias: `${patternWithNamespace}`, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index 3797b90bf8d57..de790c6633820 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -117,18 +117,19 @@ export class ConversationDataWriter implements ConversationDataWriter { const filterByUser = authenticatedUser ? [ { - bool: { - should: [ - { - term: authenticatedUser.profile_uid - ? { - 'user.id': { value: authenticatedUser.profile_uid }, - } - : { - 'user.name': { value: authenticatedUser.username }, - }, + nested: { + path: 'users', + query: { + bool: { + must: [ + { + match: authenticatedUser.profile_uid + ? { 'users.id': authenticatedUser.profile_uid } + : { 'users.name': authenticatedUser.username }, + }, + ], }, - ], + }, }, }, ] diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index 9daccb0039c65..6161eea8deff3 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -61,9 +61,11 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: '2024-01-28T04:20:02.394Z', timestamp: '2024-01-28T04:20:02.394Z', - user: { - name: 'test', - }, + users: [ + { + name: 'test', + }, + ], }); export const getSearchConversationMock = @@ -98,10 +100,12 @@ export const getSearchConversationMock = model: 'test', provider: 'Azure OpenAI', }, - user: { - id: '1111', - name: 'elastic', - }, + users: [ + { + id: '1111', + name: 'elastic', + }, + ], replacements: undefined, }, }, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index 21a67b8621f01..f0013017c07ab 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -29,13 +29,8 @@ export interface CreateMessageSchema { '@timestamp': string; content: string; reader?: Reader; - replacements?: Replacement; role: MessageRole; is_error?: boolean; - presentation?: { - delay?: number; - stream?: boolean; - }; trace_data?: { transaction_id?: string; trace_id?: string; @@ -51,10 +46,10 @@ export interface CreateMessageSchema { is_default?: boolean; exclude_from_last_conversation_storage?: boolean; replacements?: Replacement; - user: { + users: Array<{ id?: string; name?: string; - }; + }>; updated_at?: string; namespace: string; } @@ -116,10 +111,12 @@ export const transformToCreateScheme = ( return { '@timestamp': createdAt, created_at: createdAt, - user: { - id: user.profile_uid, - name: user.username, - }, + users: [ + { + id: user.profile_uid, + name: user.username, + }, + ], title, api_config: { connector_id: apiConfig?.connectorId, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts index 34eb69d740672..feba3b57ecdad 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.ts @@ -28,18 +28,19 @@ export const getConversation = async ({ const filterByUser = user ? [ { - bool: { - should: [ - { - term: user.profile_uid - ? { - 'user.id': { value: user.profile_uid }, - } - : { - 'user.name': { value: user.username }, - }, + nested: { + path: 'users', + query: { + bool: { + must: [ + { + match: user.profile_uid + ? { 'users.id': user.profile_uid } + : { 'users.name': user.username }, + }, + ], }, - ], + }, }, }, ] diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts index da3bd8ec798d5..c435c1b5feffe 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.ts @@ -20,13 +20,13 @@ import { } from '@kbn/elastic-assistant-common'; import { IIndexPatternString } from '../types'; import { ConversationDataWriter } from './conversations_data_writer'; -import { getIndexTemplateAndPattern } from '../ai_assistant_service/conversation_configuration_type'; import { createConversation } from './create_conversation'; import { findConversations } from './find_conversations'; import { updateConversation } from './update_conversation'; import { getConversation } from './get_conversation'; import { deleteConversation } from './delete_conversation'; import { appendConversationMessages } from './append_conversation_messages'; +import { getIndexTemplateAndPattern } from '../lib/data_client/helper'; export interface AIAssistantConversationsDataClientParams { elasticsearchClientPromise: Promise; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index d668cec60b0e6..398c7dd96c126 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -20,10 +20,11 @@ export const transformESToConversations = ( const conversation: ConversationResponse = { timestamp: conversationSchema['@timestamp'], createdAt: conversationSchema.created_at, - user: { - id: conversationSchema.user?.id, - name: conversationSchema.user?.name, - }, + users: + conversationSchema.users?.map((user) => ({ + id: user.id, + name: user.name, + })) ?? [], title: conversationSchema.title, apiConfig: { connectorId: conversationSchema.api_config?.connector_id, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts index 6e0fcc01aa2bd..1a65f56dad0ee 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts @@ -16,13 +16,8 @@ export interface SearchEsConversationSchema { '@timestamp': string; content: string; reader?: Reader; - replacements?: Replacement; role: MessageRole; is_error?: boolean; - presentation?: { - delay?: number; - stream?: boolean; - }; trace_data?: { transaction_id?: string; trace_id?: string; @@ -38,10 +33,10 @@ export interface SearchEsConversationSchema { is_default?: boolean; exclude_from_last_conversation_storage?: boolean; replacements?: Replacement; - user?: { + users?: Array<{ id?: string; name?: string; - }; + }>; updated_at?: string; namespace: string; } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts index 0c13163d6ce4a..6a03e07d21b46 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.test.ts @@ -55,10 +55,12 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: '2020-04-20T15:25:31.830Z', timestamp: '2020-04-20T15:25:31.830Z', - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + users: [ + { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, + ], }); jest.mock('./get_conversation', () => ({ diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index bca251fa098b6..debeddd9dc43a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -25,13 +25,8 @@ export interface UpdateConversationSchema { '@timestamp': string; content: string; reader?: Reader; - replacements?: Replacement; role: MessageRole; is_error?: boolean; - presentation?: { - delay?: number; - stream?: boolean; - }; trace_data?: { transaction_id?: string; trace_id?: string; diff --git a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts new file mode 100644 index 0000000000000..baee2da29adcf --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { + getCreateConversationSchemaMock, + getUpdateConversationSchemaMock, +} from '../../__mocks__/conversations_schema.mock'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; +import { DocumentsDataWriter } from './documents_data_writer'; + +describe('DocumentsDataWriter', () => { + const mockUser1 = { + username: 'my_username', + authentication_realm: { + type: 'my_realm_type', + name: 'my_realm_name', + }, + } as AuthenticatedUser; + describe('#bulk', () => { + let writer: DocumentsDataWriter; + let esClientMock: ElasticsearchClient; + let loggerMock: Logger; + + beforeEach(() => { + esClientMock = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; + loggerMock = loggingSystemMock.createLogger(); + writer = new DocumentsDataWriter({ + esClient: esClientMock, + logger: loggerMock, + index: 'documents-default', + spaceId: 'default', + user: { name: 'test' }, + }); + }); + + it('converts a list of documents to an appropriate list of operations', async () => { + await writer.bulk({ + documentsToCreate: [getCreateConversationSchemaMock(), getCreateConversationSchemaMock()], + documentsToUpdate: [], + documentsToDelete: [], + authenticatedUser: mockUser1, + getUpdateScript: jest.fn(), + }); + + const { docs_created: docsCreated } = (esClientMock.bulk as jest.Mock).mock.lastCall; + + expect(docsCreated).toMatchInlineSnapshot(`undefined`); + }); + + it('converts a list of mixed documents operations to an appropriate list of operations', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); + await writer.bulk({ + documentsToCreate: [getCreateConversationSchemaMock()], + documentsToUpdate: [getUpdateConversationSchemaMock()], + documentsToDelete: ['1'], + authenticatedUser: mockUser1, + getUpdateScript: jest.fn(), + }); + + const { + docs_created: docsCreated, + docs_deleted: docsDeleted, + docs_updated: docsUpdated, + } = (esClientMock.bulk as jest.Mock).mock.lastCall; + + expect(docsCreated).toMatchInlineSnapshot(`undefined`); + + expect(docsUpdated).toMatchInlineSnapshot(`undefined`); + + expect(docsDeleted).toMatchInlineSnapshot(`undefined`); + }); + + it('returns an error if something went wrong', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); + (esClientMock.bulk as jest.Mock).mockRejectedValue(new Error('something went wrong')); + + const { errors } = await writer.bulk({ + documentsToCreate: [], + documentsToUpdate: [], + documentsToDelete: ['1'], + getUpdateScript: jest.fn(), + }); + + expect(errors).toEqual([ + { + conversation: { + id: '', + }, + message: 'something went wrong', + }, + ]); + }); + + it('returns the time it took to write the documents', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + took: 123, + items: [], + }); + + const { took } = await writer.bulk({ + documentsToCreate: [], + documentsToUpdate: [], + documentsToDelete: ['1'], + getUpdateScript: jest.fn(), + }); + + expect(took).toEqual(123); + }); + + it('returns the array of docs deleted', async () => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + items: [{ delete: { status: 201 } }, { delete: { status: 200 } }], + }); + + const { docs_deleted: docsDeleted } = await writer.bulk({ + documentsToCreate: [], + documentsToUpdate: [], + documentsToDelete: ['1', '2'], + getUpdateScript: jest.fn(), + }); + + expect(docsDeleted.length).toEqual(2); + }); + + describe('when some documents failed to be written', () => { + beforeEach(() => { + (esClientMock.search as jest.Mock).mockResolvedValue({ + hits: { hits: [] }, + }); + (esClientMock.bulk as jest.Mock).mockResolvedValue({ + errors: true, + items: [ + { create: { status: 201 } }, + { create: { status: 500, error: { reason: 'something went wrong' } } }, + ], + }); + }); + + it('returns the number of docs written', async () => { + const { docs_created: docsCreated } = await writer.bulk({ + documentsToCreate: [getCreateConversationSchemaMock()], + documentsToUpdate: [], + documentsToDelete: [], + authenticatedUser: mockUser1, + getUpdateScript: jest.fn(), + }); + + expect(docsCreated.length).toEqual(1); + }); + + it('returns the errors', async () => { + const { errors } = await writer.bulk({ + documentsToCreate: [], + documentsToUpdate: [], + documentsToDelete: ['1'], + getUpdateScript: jest.fn(), + }); + + expect(errors).toEqual([ + { + conversation: { + id: undefined, + }, + message: 'something went wrong', + status: 500, + }, + ]); + }); + }); + + describe('when there are no documents to update', () => { + it('returns an appropriate response', async () => { + const response = await writer.bulk({ + getUpdateScript: jest.fn(), + }); + expect(response).toEqual({ + errors: [], + docs_created: [], + docs_deleted: [], + docs_updated: [], + took: 0, + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts new file mode 100644 index 0000000000000..7211550d97bba --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts @@ -0,0 +1,312 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidV4 } from 'uuid'; +import type { + BulkOperationContainer, + BulkOperationType, + BulkResponseItem, + Script, +} from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { UUID } from '@kbn/elastic-assistant-common'; +import { AuthenticatedUser } from '@kbn/security-plugin-types-common'; + +export interface BulkOperationError { + message: string; + status?: number; + document: { + id: string; + }; +} + +interface WriterBulkResponse { + errors: BulkOperationError[]; + docs_created: string[]; + docs_deleted: string[]; + docs_updated: string[]; + took: number; +} + +interface BulkParams { + documentsToCreate?: TCreateParams[]; + documentsToUpdate?: TUpdateParams[]; + documentsToDelete?: string[]; + getUpdateScript: (document: TUpdateParams, updatedAt: string) => Script; + authenticatedUser?: AuthenticatedUser; +} + +export interface DocumentsDataWriter { + bulk: ( + params: BulkParams + ) => Promise; +} + +interface DocumentsDataWriterOptions { + esClient: ElasticsearchClient; + index: string; + spaceId: string; + user: { id?: UUID; name?: string }; + logger: Logger; +} + +export class DocumentsDataWriter implements DocumentsDataWriter { + constructor(private readonly options: DocumentsDataWriterOptions) {} + + public bulk = async ( + params: BulkParams + ) => { + try { + if ( + !params.documentsToCreate?.length && + !params.documentsToUpdate?.length && + !params.documentsToDelete?.length + ) { + return { errors: [], docs_created: [], docs_deleted: [], docs_updated: [], took: 0 }; + } + + const { errors, items, took } = await this.options.esClient.bulk({ + refresh: 'wait_for', + body: await this.buildBulkOperations(params), + }); + + return { + errors: errors ? this.formatErrorsResponse(items) : [], + docs_created: items + .filter((item) => item.create?.status === 201 || item.create?.status === 200) + .map((item) => item.create?._id ?? ''), + docs_deleted: items + .filter((item) => item.delete?.status === 201 || item.delete?.status === 200) + .map((item) => item.delete?._id ?? ''), + docs_updated: items + .filter((item) => item.update?.status === 201 || item.update?.status === 200) + .map((item) => item.update?._id ?? ''), + took, + } as WriterBulkResponse; + } catch (e) { + this.options.logger.error(`Error bulk actions for documents: ${e.message}`); + return { + errors: [ + { + message: e.message, + document: { + id: '', + }, + }, + ], + docs_created: [], + docs_deleted: [], + docs_updated: [], + took: 0, + } as WriterBulkResponse; + } + }; + + private getUpdateDocumentsQuery = async ( + documentsToUpdate: TUpdateParams[], + getUpdateScript: (document: TUpdateParams, updatedAt: string) => Script, + authenticatedUser?: AuthenticatedUser + ) => { + const updatedAt = new Date().toISOString(); + const filterByUser = authenticatedUser + ? [ + { + bool: { + should: [ + { + term: authenticatedUser.profile_uid + ? { + 'user.id': { value: authenticatedUser.profile_uid }, + } + : { + 'user.name': { value: authenticatedUser.username }, + }, + }, + ], + }, + }, + ] + : []; + + const responseToUpdate = await this.options.esClient.search({ + body: { + query: { + bool: { + must: [ + { + bool: { + should: [ + { + ids: { + values: documentsToUpdate?.map((c) => c.id), + }, + }, + ], + }, + }, + ...filterByUser, + ], + }, + }, + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }); + + const availableDocumentsToUpdate = documentsToUpdate.filter((c) => + responseToUpdate?.hits.hits.find((ac) => ac._id === c.id) + ); + + return availableDocumentsToUpdate.flatMap((document) => [ + { + update: { + _id: document.id, + _index: responseToUpdate?.hits.hits.find((c) => c._id === document.id)?._index, + }, + }, + { + script: getUpdateScript(document, updatedAt), + upsert: { counter: 1 }, + }, + ]); + }; + + private getDeletedocumentsQuery = async ( + documentsToDelete: string[], + authenticatedUser?: AuthenticatedUser + ) => { + const filterByUser = authenticatedUser + ? [ + { + nested: { + path: 'users', + query: { + bool: { + must: [ + { + match: authenticatedUser.profile_uid + ? { 'users.id': authenticatedUser.profile_uid } + : { 'users.name': authenticatedUser.username }, + }, + ], + }, + }, + }, + }, + ] + : []; + + const responseToDelete = await this.options.esClient.search({ + body: { + query: { + bool: { + must: [ + { + bool: { + should: [ + { + ids: { + values: documentsToDelete, + }, + }, + ], + }, + }, + ...filterByUser, + ], + }, + }, + }, + _source: false, + ignore_unavailable: true, + index: this.options.index, + seq_no_primary_term: true, + size: 1000, + }); + + return ( + responseToDelete?.hits.hits.map((c) => [ + { + delete: { + _id: c._id, + _index: c._index, + }, + }, + ]) ?? [] + ); + }; + + private buildBulkOperations = async ( + params: BulkParams + ): Promise => { + const changedAt = new Date().toISOString(); + const documentCreateBody = + params.authenticatedUser && params.documentsToCreate + ? params.documentsToCreate.flatMap((document) => [ + { create: { _index: this.options.index, _id: uuidV4() } }, + document, + ]) + : []; + + const documentDeletedBody = + params.documentsToDelete && params.documentsToDelete.length > 0 + ? await this.getDeletedocumentsQuery(params.documentsToDelete, params.authenticatedUser) + : []; + + const documentUpdatedBody = + params.documentsToUpdate && params.documentsToUpdate.length > 0 + ? await this.getUpdateDocumentsQuery( + params.documentsToUpdate, + params.getUpdateScript, + params.authenticatedUser + ) + : []; + + return [ + ...documentCreateBody, + ...documentUpdatedBody, + ...documentDeletedBody, + ] as BulkOperationContainer[]; + }; + + private formatErrorsResponse = ( + items: Array>> + ) => { + return items + .map((item) => + item.create?.error + ? { + message: item.create.error?.reason, + status: item.create.status, + document: { + id: item.create._id, + }, + } + : item.update?.error + ? { + message: item.update.error?.reason, + status: item.update.status, + document: { + id: item.update._id, + }, + } + : item.delete?.error + ? { + message: item.delete?.error?.reason, + status: item.delete?.status, + document: { + id: item.delete?._id, + }, + } + : undefined + ) + .filter((e) => e !== undefined); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/lib/data_client/helper.ts b/x-pack/plugins/elastic_assistant/server/lib/data_client/helper.ts new file mode 100644 index 0000000000000..6f454ed3d4b56 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/data_client/helper.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { IIndexPatternString } from '../../types'; + +export const getIndexTemplateAndPattern = ( + context: string, + namespace?: string +): IIndexPatternString => { + const concreteNamespace = namespace ? namespace : DEFAULT_NAMESPACE_STRING; + const pattern = `${context}`; + const patternWithNamespace = `${pattern}-${concreteNamespace}`; + return { + pattern: `${patternWithNamespace}*`, + basePattern: `${pattern}-*`, + name: `${patternWithNamespace}-000001`, + alias: `${patternWithNamespace}`, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/plugin.ts b/x-pack/plugins/elastic_assistant/server/plugin.ts index 75ade90d7ea67..53b05857beb4b 100755 --- a/x-pack/plugins/elastic_assistant/server/plugin.ts +++ b/x-pack/plugins/elastic_assistant/server/plugin.ts @@ -20,7 +20,6 @@ import { ElasticAssistantRequestHandlerContext, } from './types'; import { AIAssistantService } from './ai_assistant_service'; -import { assistantPromptsType, assistantAnonymizationFieldsType } from './saved_object'; import { RequestContextFactory } from './routes/request_context_factory'; import { PLUGIN_ID } from '../common/constants'; import { registerRoutes } from './routes/register_routes'; @@ -77,9 +76,6 @@ export class ElasticAssistantPlugin ); events.forEach((eventConfig) => core.analytics.registerEventType(eventConfig)); - core.savedObjects.registerType(assistantPromptsType); - core.savedObjects.registerType(assistantAnonymizationFieldsType); - // this.assistantService registerKBTask registerRoutes(router, this.logger, plugins); return { diff --git a/x-pack/plugins/elastic_assistant/server/promts_data_client/find_prompts.ts b/x-pack/plugins/elastic_assistant/server/promts_data_client/find_prompts.ts new file mode 100644 index 0000000000000..e8b73cc5b8fa6 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/promts_data_client/find_prompts.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingRuntimeFields, Sort } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import { estypes } from '@elastic/elasticsearch'; +import { EsQueryConfig, Query, buildEsQuery } from '@kbn/es-query'; +import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; +import { SearchEsPromptsSchema } from './types'; +import { transformESToPrompts } from './helpers'; + +interface FindPromptsOptions { + filter?: string; + fields?: string[]; + perPage: number; + page: number; + sortField?: string; + sortOrder?: estypes.SortOrder; + esClient: ElasticsearchClient; + anonymizationFieldsIndex: string; + runtimeMappings?: MappingRuntimeFields | undefined; +} + +export const findPrompts = async ({ + esClient, + filter, + page, + perPage, + sortField, + anonymizationFieldsIndex, + fields, + sortOrder, +}: FindPromptsOptions): Promise => { + const query = getQueryFilter({ filter }); + let sort: Sort | undefined; + const ascOrDesc = sortOrder ?? ('asc' as const); + if (sortField != null) { + sort = [{ [sortField]: ascOrDesc }]; + } else { + sort = { + updated_at: { + order: 'desc', + }, + }; + } + const response = await esClient.search({ + body: { + query, + track_total_hits: true, + sort, + }, + _source: true, + from: (page - 1) * perPage, + ignore_unavailable: true, + index: anonymizationFieldsIndex, + seq_no_primary_term: true, + size: perPage, + }); + return { + data: transformESToPrompts(response), + page, + perPage, + total: + (typeof response.hits.total === 'number' + ? response.hits.total // This format is to be removed in 8.0 + : response.hits.total?.value) ?? 0, + }; +}; + +export interface GetQueryFilterOptions { + filter?: string; +} + +export const getQueryFilter = ({ filter }: GetQueryFilterOptions) => { + const kqlQuery: Query | Query[] = filter + ? { + language: 'kuery', + query: filter, + } + : []; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + dateFormatTZ: 'Zulu', + ignoreFilterIfFieldNotInIndex: false, + queryStringOptions: { analyze_wildcard: true }, + }; + + return buildEsQuery(undefined, kqlQuery, [], config); +}; diff --git a/x-pack/plugins/elastic_assistant/server/promts_data_client/helpers.ts b/x-pack/plugins/elastic_assistant/server/promts_data_client/helpers.ts new file mode 100644 index 0000000000000..3544635ebbebd --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/promts_data_client/helpers.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { + PromptResponse, + PromptUpdateProps, +} from '@kbn/elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen'; +import { SearchEsPromptsSchema, UpdatePromptSchema } from './types'; + +export const transformESToPrompts = ( + response: estypes.SearchResponse +): PromptResponse[] => { + return response.hits.hits + .filter((hit) => hit._source !== undefined) + .map((hit) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const promptSchema = hit._source!; + const prompt: PromptResponse = { + timestamp: promptSchema['@timestamp'], + createdAt: promptSchema.created_at, + users: + promptSchema.users?.map((user) => ({ + id: user.id, + name: user.name, + })) ?? [], + content: promptSchema.content, + isDefault: promptSchema.is_default, + isNewConversationDefault: promptSchema.is_new_conversation_default, + updatedAt: promptSchema.updated_at, + namespace: promptSchema.namespace, + id: hit._id, + name: promptSchema.name, + promptType: promptSchema.prompt_type, + isShared: promptSchema.is_shared, + createdBy: promptSchema.created_by, + updatedBy: promptSchema.updated_by, + }; + + return prompt; + }); +}; + +export const transformToUpdateScheme = ( + updatedAt: string, + { content, isDefault, isNewConversationDefault, isShared }: PromptUpdateProps +): UpdatePromptSchema => { + return { + updated_at: updatedAt, + content: content ?? '', + is_new_conversation_default: isNewConversationDefault, + is_shared: isShared, + }; +}; + +export const getUpdateScript = ({ + prompt, + updatedAt, + isPatch, +}: { + prompt: PromptUpdateProps; + updatedAt: string; + isPatch?: boolean; +}) => { + return { + source: ` + if (params.assignEmpty == true || params.containsKey('content')) { + ctx._source.content = params.content; + } + if (params.assignEmpty == true || params.containsKey('is_new_conversation_default')) { + ctx._source.is_new_conversation_default = params.is_new_conversation_default; + } + if (params.assignEmpty == true || params.containsKey('is_shared')) { + ctx._source.is_shared = params.is_shared; + } + ctx._source.updated_at = params.updated_at; + `, + lang: 'painless', + params: { + ...transformToUpdateScheme(updatedAt, prompt), // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !(isPatch ?? true), + }, + }; +}; diff --git a/x-pack/plugins/elastic_assistant/server/promts_data_client/index.ts b/x-pack/plugins/elastic_assistant/server/promts_data_client/index.ts new file mode 100644 index 0000000000000..bfdab8ede6b60 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/promts_data_client/index.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; +import { AuthenticatedUser } from '@kbn/security-plugin/server'; +import { estypes } from '@elastic/elasticsearch'; + +import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; +import { PromptResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen'; + +import { DocumentsDataWriter } from '../lib/data_client/documents_data_writer'; +import { IIndexPatternString } from '../types'; +import { getIndexTemplateAndPattern } from '../lib/data_client/helper'; +import { findPrompts } from './find_prompts'; + +export interface AIAssistantPromtsDataClientParams { + elasticsearchClientPromise: Promise; + kibanaVersion: string; + spaceId: string; + logger: Logger; + indexPatternsResorceName: string; + currentUser: AuthenticatedUser | null; +} + +/** + * Class for use for prompts that are used for AI assistant. + */ +export class AIAssistantPromtsDataClient { + /** Kibana space id the anonymization fields are part of */ + private readonly spaceId: string; + + /** User creating, modifying, deleting, or updating a anonymization fields */ + private readonly currentUser: AuthenticatedUser | null; + + private writerCache: Map = new Map(); + + private indexTemplateAndPattern: IIndexPatternString; + + constructor(private readonly options: AIAssistantPromtsDataClientParams) { + this.indexTemplateAndPattern = getIndexTemplateAndPattern( + this.options.indexPatternsResorceName, + this.options.spaceId ?? DEFAULT_NAMESPACE_STRING + ); + this.currentUser = this.options.currentUser; + this.spaceId = this.options.spaceId; + } + + public getWriter = async (): Promise => { + const spaceId = this.spaceId; + if (this.writerCache.get(spaceId)) { + return this.writerCache.get(spaceId) as DocumentsDataWriter; + } + await this.initializeWriter(spaceId, this.indexTemplateAndPattern.alias); + return this.writerCache.get(spaceId) as DocumentsDataWriter; + }; + + private async initializeWriter(spaceId: string, index: string): Promise { + const esClient = await this.options.elasticsearchClientPromise; + const writer = new DocumentsDataWriter({ + esClient, + spaceId, + index, + logger: this.options.logger, + user: { id: this.currentUser?.profile_uid, name: this.currentUser?.username }, + }); + + this.writerCache.set(spaceId, writer); + return writer; + } + + public getReader = async (options: { spaceId?: string } = {}) => { + const indexPatterns = this.indexTemplateAndPattern.alias; + + return { + search: async < + TSearchRequest extends ESSearchRequest, + TAnonymizationFieldDoc = Partial + >( + request: TSearchRequest + ): Promise> => { + try { + const esClient = await this.options.elasticsearchClientPromise; + return (await esClient.search({ + ...request, + index: indexPatterns, + ignore_unavailable: true, + seq_no_primary_term: true, + })) as unknown as ESSearchResponse; + } catch (err) { + this.options.logger.error( + `Error performing search in AIAssistantDataClient - ${err.message}` + ); + throw err; + } + }, + }; + }; + + public findPrompts = async ({ + perPage, + page, + sortField, + sortOrder, + filter, + fields, + }: { + perPage: number; + page: number; + sortField?: string; + sortOrder?: string; + filter?: string; + fields?: string[]; + }): Promise => { + const esClient = await this.options.elasticsearchClientPromise; + return findPrompts({ + esClient, + fields, + page, + perPage, + filter, + sortField, + anonymizationFieldsIndex: this.indexTemplateAndPattern.alias, + sortOrder: sortOrder as estypes.SortOrder, + }); + }; +} diff --git a/x-pack/plugins/elastic_assistant/server/promts_data_client/prompts_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/promts_data_client/prompts_configuration_type.ts new file mode 100644 index 0000000000000..50df573d01872 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/promts_data_client/prompts_configuration_type.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldMap } from '@kbn/data-stream-adapter'; + +export const assistantPromptsFieldMap: FieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: false, + }, + id: { + type: 'keyword', + array: false, + required: true, + }, + is_default: { + type: 'boolean', + array: false, + required: false, + }, + is_shared: { + type: 'boolean', + array: false, + required: false, + }, + is_new_conversation_default: { + type: 'boolean', + array: false, + required: false, + }, + name: { + type: 'keyword', + array: false, + required: true, + }, + prompt_type: { + type: 'keyword', + array: false, + required: true, + }, + content: { + type: 'keyword', + array: false, + required: true, + }, + updated_at: { + type: 'date', + array: false, + required: false, + }, + updated_by: { + type: 'keyword', + array: false, + required: false, + }, + created_at: { + type: 'date', + array: false, + required: false, + }, + created_by: { + type: 'keyword', + array: false, + required: false, + }, + users: { + type: 'nested', + array: true, + required: false, + }, + 'users.id': { + type: 'keyword', + array: false, + required: false, + }, + 'users.name': { + type: 'keyword', + array: false, + required: false, + }, +}; diff --git a/x-pack/plugins/elastic_assistant/server/promts_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/promts_data_client/types.ts new file mode 100644 index 0000000000000..6800cd3f2a9fa --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/promts_data_client/types.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface SearchEsPromptsSchema { + id: string; + '@timestamp': string; + created_at: string; + created_by: string; + content: string; + is_default?: boolean; + is_shared?: boolean; + is_new_conversation_default?: boolean; + name: string; + prompt_type: string; + users?: Array<{ + id?: string; + name?: string; + }>; + updated_at?: string; + updated_by?: string; + namespace: string; +} + +export interface UpdatePromptSchema { + '@timestamp'?: string; + is_shared?: boolean; + is_new_conversation_default?: boolean; + content: string; + updated_at?: string; + updated_by?: string; +} diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts index 83520b06fcd05..2cd62494e0ac8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/bulk_actions_route.ts @@ -14,9 +14,9 @@ import { ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, } from '@kbn/elastic-assistant-common'; -import { SavedObjectError } from '@kbn/core/types'; import { AnonymizationFieldResponse, + AnonymizationFieldUpdateProps, BulkActionSkipResult, BulkCrudActionResponse, BulkCrudActionResults, @@ -28,13 +28,13 @@ import { ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/consta import { ElasticAssistantPluginRouter } from '../../types'; import { buildRouteValidationWithZod } from '../route_validation'; import { buildResponse } from '../utils'; +import { getUpdateScript } from '../../anonymization_fields_data_client/helpers'; export interface BulkOperationError { message: string; status?: number; - anonymizationField: { + document: { id: string; - name: string; }; } @@ -49,7 +49,7 @@ const buildBulkResponse = ( deleted = [], skipped = [], }: { - errors?: SavedObjectError[]; + errors?: BulkOperationError[]; updated?: AnonymizationFieldResponse[]; created?: AnonymizationFieldResponse[]; deleted?: string[]; @@ -141,43 +141,71 @@ export const bulkActionAnonymizationFieldsRoute = ( request.events.completed$.subscribe(() => abortController.abort()); try { const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsSOClient(); - - const docsCreated = - body.create && body.create.length > 0 - ? await dataClient.createAnonymizationFields(body.create) - : []; - const docsUpdated = - body.update && body.update.length > 0 - ? await dataClient.updateAnonymizationFields(body.update) - : []; - const docsDeleted = await dataClient.deleteAnonymizationFieldsByIds( - body.delete?.ids ?? [] - ); + + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } + const dataClient = + await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsDataClient(); + + if (body.create && body.create.length > 0) { + const result = await dataClient?.findAnonymizationFields({ + perPage: 100, + page: 1, + filter: `user.id:${authenticatedUser?.profile_uid} AND (${body.create + .map((c) => `field:${c.field}`) + .join(' OR ')})`, + fields: ['field'], + }); + if (result?.data != null && result.data.length > 0) { + return assistantResponse.error({ + statusCode: 409, + body: `anonymization for field: "${result.data + .map((c) => c.field) + .join(',')}" already exists`, + }); + } + } + + const writer = await dataClient?.getWriter(); + + const { + errors, + docs_created: docsCreated, + docs_updated: docsUpdated, + docs_deleted: docsDeleted, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + } = await writer!.bulk({ + documentsToCreate: body.create, + documentsToDelete: body.delete?.ids, + documentsToUpdate: body.update, + getUpdateScript: (document: AnonymizationFieldUpdateProps, updatedAt: string) => + getUpdateScript({ anonymizationField: document, updatedAt, isPatch: false }), + authenticatedUser, + }); const created = await dataClient?.findAnonymizationFields({ page: 1, perPage: 1000, - filter: docsCreated.map((updatedId) => `id:${updatedId}`).join(' OR '), + filter: docsCreated.map((c) => `id:${c}`).join(' OR '), fields: ['id'], }); const updated = await dataClient?.findAnonymizationFields({ page: 1, perPage: 1000, - filter: docsUpdated.map((updatedId) => `id:${updatedId}`).join(' OR '), + filter: docsUpdated.map((c) => `id:${c}`).join(' OR '), fields: ['id'], }); return buildBulkResponse(response, { updated: updated?.data, created: created?.data, - deleted: docsDeleted.map((d) => d.id) ?? [], - errors: docsDeleted.reduce((res, d) => { - if (d.error !== undefined) { - res.push(d.error); - } - return res; - }, [] as SavedObjectError[]), + deleted: docsDeleted ?? [], + errors, }); } catch (err) { const error = transformError(err); diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts index a2de850544be8..b71926a1b0dbc 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/find_route.ts @@ -52,7 +52,8 @@ export const findAnonymizationFieldsRoute = ( try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsSOClient(); + const dataClient = + await ctx.elasticAssistant.getAIAssistantAnonymizationFieldsDataClient(); const result = await dataClient?.findAnonymizationFields({ perPage: query.per_page, diff --git a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts deleted file mode 100644 index 45dd9bf5c09ae..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/anonimization_fields/update_route.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IKibanaResponse } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; -import { - ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, -} from '@kbn/elastic-assistant-common'; -import { - PromptResponse, - PromptUpdateProps, -} from '@kbn/elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; - -export const updatePromptRoute = (router: ElasticAssistantPluginRouter) => { - router.versioned - .put({ - access: 'public', - path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: { - request: { - body: buildRouteValidationWithZod(PromptUpdateProps), - params: schema.object({ - promptId: schema.string(), - }), - }, - }, - }, - async (context, request, response): Promise> => { - const assistantResponse = buildResponse(response); - const { promptId } = request.params; - - try { - const ctx = await context.resolve(['core', 'elasticAssistant']); - - const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); - - const existingPrompt = await dataClient?.getPrompt(promptId); - if (existingPrompt == null) { - return assistantResponse.error({ - body: `Prompt id: "${promptId}" not found`, - statusCode: 404, - }); - } - const prompt = await dataClient?.updatePromptItem(existingPrompt, request.body); - if (prompt == null) { - return assistantResponse.error({ - body: `prompt id: "${promptId}" was not updated`, - statusCode: 400, - }); - } - return response.ok({ - body: prompt, - }); - } catch (err) { - const error = transformError(err); - return assistantResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/bulk_actions_route.ts new file mode 100644 index 0000000000000..fb8e11dbbe744 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/bulk_actions_route.ts @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { + ELASTIC_AI_ASSISTANT_PROMPTS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; + +import { + PromptResponse, + BulkActionSkipResult, + BulkCrudActionResponse, + BulkCrudActionResults, + BulkCrudActionSummary, + PerformBulkActionRequestBody, + PerformBulkActionResponse, + PromptUpdateProps, +} from '@kbn/elastic-assistant-common/impl/schemas/prompts/bulk_crud_prompts_route.gen'; +import { ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE } from '../../../common/constants'; +import { ElasticAssistantPluginRouter } from '../../types'; +import { buildRouteValidationWithZod } from '../route_validation'; +import { buildResponse } from '../utils'; +import { getUpdateScript } from '../../promts_data_client/helpers'; + +export interface BulkOperationError { + message: string; + status?: number; + document: { + id: string; + }; +} + +export type BulkActionError = BulkOperationError | unknown; + +const buildBulkResponse = ( + response: KibanaResponseFactory, + { + errors = [], + updated = [], + created = [], + deleted = [], + skipped = [], + }: { + errors?: BulkOperationError[]; + updated?: PromptResponse[]; + created?: PromptResponse[]; + deleted?: string[]; + skipped?: BulkActionSkipResult[]; + } +): IKibanaResponse => { + const numSucceeded = updated.length + created.length + deleted.length; + const numSkipped = skipped.length; + const numFailed = errors.length; + + const summary: BulkCrudActionSummary = { + failed: numFailed, + succeeded: numSucceeded, + skipped: numSkipped, + total: numSucceeded + numFailed + numSkipped, + }; + + const results: BulkCrudActionResults = { + updated, + created, + deleted, + skipped, + }; + + if (numFailed > 0) { + return response.custom({ + headers: { 'content-type': 'application/json' }, + body: { + message: summary.succeeded > 0 ? 'Bulk edit partially failed' : 'Bulk edit failed', + status_code: 500, + attributes: { + errors: [], + results, + summary, + }, + }, + statusCode: 500, + }); + } + + const responseBody: BulkCrudActionResponse = { + success: true, + prompts_count: summary.total, + attributes: { results, summary }, + }; + + return response.ok({ body: responseBody }); +}; + +export const bulkPromptsRoute = (router: ElasticAssistantPluginRouter, logger: Logger) => { + router.versioned + .post({ + access: 'public', + path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BULK_ACTION, + options: { + tags: ['access:elasticAssistant'], + timeout: { + idleSocket: moment.duration(15, 'minutes').asMilliseconds(), + }, + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + validate: { + request: { + body: buildRouteValidationWithZod(PerformBulkActionRequestBody), + }, + }, + }, + async (context, request, response): Promise> => { + const { body } = request; + const assistantResponse = buildResponse(response); + + if (body?.update && body.update?.length > ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE) { + return assistantResponse.error({ + body: `More than ${ANONYMIZATION_FIELDS_TABLE_MAX_PAGE_SIZE} ids sent for bulk edit action.`, + statusCode: 400, + }); + } + + const abortController = new AbortController(); + + // subscribing to completed$, because it handles both cases when request was completed and aborted. + // when route is finished by timeout, aborted$ is not getting fired + request.events.completed$.subscribe(() => abortController.abort()); + try { + const ctx = await context.resolve(['core', 'elasticAssistant']); + + const authenticatedUser = ctx.elasticAssistant.getCurrentUser(); + if (authenticatedUser == null) { + return assistantResponse.error({ + body: `Authenticated user not found`, + statusCode: 401, + }); + } + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsDataClient(); + + if (body.create && body.create.length > 0) { + const result = await dataClient?.findPrompts({ + perPage: 100, + page: 1, + filter: `user.id:${authenticatedUser?.profile_uid} AND (${body.create + .map((c) => `name:${c.name}`) + .join(' OR ')})`, + fields: ['name'], + }); + if (result?.data != null && result.data.length > 0) { + return assistantResponse.error({ + statusCode: 409, + body: `prompt with name: "${result.data + .map((c) => c.name) + .join(',')}" already exists`, + }); + } + } + + const writer = await dataClient?.getWriter(); + + const { + errors, + docs_created: docsCreated, + docs_updated: docsUpdated, + docs_deleted: docsDeleted, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + } = await writer!.bulk({ + documentsToCreate: body.create, + documentsToDelete: body.delete?.ids, + documentsToUpdate: body.update, + getUpdateScript: (document: PromptUpdateProps, updatedAt: string) => + getUpdateScript({ prompt: document, updatedAt, isPatch: false }), + authenticatedUser, + }); + + const created = await dataClient?.findPrompts({ + page: 1, + perPage: 1000, + filter: docsCreated.map((c) => `id:${c}`).join(' OR '), + fields: ['id'], + }); + const updated = await dataClient?.findPrompts({ + page: 1, + perPage: 1000, + filter: docsUpdated.map((c) => `id:${c}`).join(' OR '), + fields: ['id'], + }); + + return buildBulkResponse(response, { + updated: updated?.data, + created: created?.data, + deleted: docsDeleted ?? [], + errors, + }); + } catch (err) { + const error = transformError(err); + return assistantResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts deleted file mode 100644 index ea776bc7c2d69..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/create_route.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IKibanaResponse } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { - PromptCreateProps, - PromptResponse, - ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_PROMPTS_URL, -} from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildResponse } from '../utils'; -import { buildRouteValidationWithZod } from '../route_validation'; - -export const createPromptRoute = (router: ElasticAssistantPluginRouter): void => { - router.versioned - .post({ - access: 'public', - path: ELASTIC_AI_ASSISTANT_PROMPTS_URL, - - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: { - request: { - body: buildRouteValidationWithZod(PromptCreateProps), - }, - }, - }, - async (context, request, response): Promise> => { - const assistantResponse = buildResponse(response); - // const validationErrors = validateCreateRuleProps(request.body); - // if (validationErrors.length) { - // return siemResponse.error({ statusCode: 400, body: validationErrors }); - // } - - try { - const ctx = await context.resolve(['core', 'elasticAssistant']); - - const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); - const createdPrompt = await dataClient.createPrompt(request.body); - return response.ok({ - body: PromptResponse.parse(createdPrompt), - }); - } catch (err) { - const error = transformError(err as Error); - return assistantResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts deleted file mode 100644 index dd8dbd873dc27..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/delete_route.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IKibanaResponse } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; -import { - ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, -} from '@kbn/elastic-assistant-common'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildResponse } from '../utils'; - -export const deletePromptRoute = (router: ElasticAssistantPluginRouter) => { - router.versioned - .delete({ - access: 'public', - path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: { - request: { - params: schema.object({ - promptId: schema.string(), - }), - }, - }, - }, - async (context, request, response): Promise => { - const assistantResponse = buildResponse(response); - /* const validationErrors = validateQueryRuleByIds(request.query); - if (validationErrors.length) { - return siemResponse.error({ statusCode: 400, body: validationErrors }); - }*/ - - try { - const { promptId } = request.params; - - const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); - - const existingPrompt = await dataClient?.getPrompt(promptId); - if (existingPrompt == null) { - return assistantResponse.error({ - body: `prompt id: "${promptId}" not found`, - statusCode: 404, - }); - } - await dataClient?.deletePromptById(promptId); - - return response.ok({ body: {} }); - } catch (err) { - const error = transformError(err); - return assistantResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts index 8eb7c2f291f52..39e8c16b09335 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/prompts/find_route.ts @@ -44,7 +44,7 @@ export const findPromptsRoute = (router: ElasticAssistantPluginRouter, logger: L try { const { query } = request; const ctx = await context.resolve(['core', 'elasticAssistant']); - const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); + const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsDataClient(); const result = await dataClient?.findPrompts({ perPage: query.per_page, diff --git a/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts b/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts deleted file mode 100644 index 45dd9bf5c09ae..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/prompts/update_route.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IKibanaResponse } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { schema } from '@kbn/config-schema'; -import { - ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, -} from '@kbn/elastic-assistant-common'; -import { - PromptResponse, - PromptUpdateProps, -} from '@kbn/elastic-assistant-common/impl/schemas/prompts/crud_prompts_route.gen'; -import { ElasticAssistantPluginRouter } from '../../types'; -import { buildRouteValidationWithZod } from '../route_validation'; -import { buildResponse } from '../utils'; - -export const updatePromptRoute = (router: ElasticAssistantPluginRouter) => { - router.versioned - .put({ - access: 'public', - path: ELASTIC_AI_ASSISTANT_PROMPTS_URL_BY_ID, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - validate: { - request: { - body: buildRouteValidationWithZod(PromptUpdateProps), - params: schema.object({ - promptId: schema.string(), - }), - }, - }, - }, - async (context, request, response): Promise> => { - const assistantResponse = buildResponse(response); - const { promptId } = request.params; - - try { - const ctx = await context.resolve(['core', 'elasticAssistant']); - - const dataClient = await ctx.elasticAssistant.getAIAssistantPromptsSOClient(); - - const existingPrompt = await dataClient?.getPrompt(promptId); - if (existingPrompt == null) { - return assistantResponse.error({ - body: `Prompt id: "${promptId}" not found`, - statusCode: 404, - }); - } - const prompt = await dataClient?.updatePromptItem(existingPrompt, request.body); - if (prompt == null) { - return assistantResponse.error({ - body: `prompt id: "${promptId}" was not updated`, - statusCode: 400, - }); - } - return response.ok({ - body: prompt, - }); - } catch (err) { - const error = transformError(err); - return assistantResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index 88453221f5bdd..eeff165a67830 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -26,10 +26,10 @@ import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; import { postEvaluateRoute } from './evaluate/post_evaluate'; import { postActionsConnectorExecuteRoute } from './post_actions_connector_execute'; import { getCapabilitiesRoute } from './capabilities/get_capabilities_route'; -import { createPromptRoute } from './prompts/create_route'; -import { updatePromptRoute } from './prompts/update_route'; -import { deletePromptRoute } from './prompts/delete_route'; +import { bulkPromptsRoute } from './prompts/bulk_actions_route'; import { findPromptsRoute } from './prompts/find_route'; +import { bulkActionAnonymizationFieldsRoute } from './anonimization_fields/bulk_actions_route'; +import { findAnonymizationFieldsRoute } from './anonimization_fields/find_route'; export const registerRoutes = ( router: ElasticAssistantPluginRouter, @@ -70,8 +70,10 @@ export const registerRoutes = ( postEvaluateRoute(router, getElserId); // Prompts - createPromptRoute(router); + bulkPromptsRoute(router, logger); findPromptsRoute(router, logger); - updatePromptRoute(router); - deletePromptRoute(router); + + // Anonymization Fields + bulkActionAnonymizationFieldsRoute(router, logger); + findAnonymizationFieldsRoute(router, logger); }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 440340db9747f..82e21a8cd8690 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -16,10 +16,8 @@ import { ElasticAssistantPluginSetupDependencies, ElasticAssistantRequestHandlerContext, } from '../types'; -import { AIAssistantPromptsSOClient } from '../saved_object/ai_assistant_prompts_so_client'; import { AIAssistantService } from '../ai_assistant_service'; import { appContextService } from '../services/app_context'; -import { AIAssistantAnonymizationFieldsSOClient } from '../saved_object/ai_assistant_anonymization_fields_so_client'; export interface IRequestContextFactory { create( @@ -83,29 +81,27 @@ export class RequestContextFactory implements IRequestContextFactory { telemetry: core.analytics, - getAIAssistantPromptsSOClient: memoize(() => { - const username = - startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; - return new AIAssistantPromptsSOClient({ - logger: options.logger, - user: username, - savedObjectsClient: coreContext.savedObjects.client, + getAIAssistantPromptsDataClient: memoize(() => { + const currentUser = getCurrentUser(); + return this.assistantService.createAIAssistantPromptsDataClient({ + spaceId: getSpaceId(), + logger: this.logger, + currentUser, }); }), - getAIAssistantAnonymizationFieldsSOClient: memoize(() => { - const username = - startPlugins.security?.authc.getCurrentUser(request)?.username || 'elastic'; - return new AIAssistantAnonymizationFieldsSOClient({ - logger: options.logger, - user: username, - savedObjectsClient: coreContext.savedObjects.client, + getAIAssistantAnonymizationFieldsDataClient: memoize(() => { + const currentUser = getCurrentUser(); + return this.assistantService.createAIAssistantAnonymizationFieldsDataClient({ + spaceId: getSpaceId(), + logger: this.logger, + currentUser, }); }), getAIAssistantConversationsDataClient: memoize(async () => { const currentUser = getCurrentUser(); - return this.assistantService.createAIAssistantDatastreamClient({ + return this.assistantService.createAIAssistantConversationsDataClient({ spaceId: getSpaceId(), logger: this.logger, currentUser, diff --git a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts index ac522174c12bd..5c845e90b38fb 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts @@ -52,7 +52,7 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) page: query.page, sortField: query.sort_field, sortOrder: query.sort_order, - filter: `user.id:${currentUser?.profile_uid}${additionalFilter}`, + filter: `users:{ id: "${currentUser?.profile_uid}${additionalFilter}" }`, fields: query.fields, }); diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts deleted file mode 100644 index b88fe52f2b893..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_anonymization_fields_so_client.ts +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - Logger, - SavedObjectsErrorHelpers, - type SavedObjectsClientContract, - SavedObjectsBulkDeleteStatus, -} from '@kbn/core/server'; - -import { - AnonymizationFieldCreateProps, - AnonymizationFieldResponse, - AnonymizationFieldUpdateProps, -} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; -import { - FindAnonymizationFieldsResponse, - SortOrder, -} from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; -import { - AssistantAnonymizationFieldSoSchema, - assistantAnonymizationFieldsTypeName, - transformSavedObjectToAssistantAnonymizationField, - transformSavedObjectUpdateToAssistantAnonymizationField, - transformSavedObjectsToFoundAssistantAnonymizationField, -} from './elastic_assistant_anonymization_fields_type'; - -export interface ConstructorOptions { - /** User creating, modifying, deleting, or updating the anonymization fields */ - user: string; - /** Saved objects client to create, modify, delete, the anonymization fields */ - savedObjectsClient: SavedObjectsClientContract; - logger: Logger; -} - -/** - * Class for use for anonymization fields that are used for AI assistant. - */ -export class AIAssistantAnonymizationFieldsSOClient { - /** User creating, modifying, deleting, or updating the anonymization fields */ - private readonly user: string; - - /** Saved objects client to create, modify, delete, the anonymization fields */ - private readonly savedObjectsClient: SavedObjectsClientContract; - - /** - * Constructs the assistant client - * @param options - * @param options.user The user associated with the anonymization fields - * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI anonymization fields - */ - constructor({ user, savedObjectsClient }: ConstructorOptions) { - this.user = user; - this.savedObjectsClient = savedObjectsClient; - } - - /** - * Fetch an anonymization field - * @param options - * @param options.id the "id" of an exception list - * @returns The found exception list or null if none exists - */ - public getAnonymizationField = async (id: string): Promise => { - const { savedObjectsClient } = this; - if (id != null) { - try { - const savedObject = await savedObjectsClient.get( - assistantAnonymizationFieldsTypeName, - id - ); - return transformSavedObjectToAssistantAnonymizationField({ savedObject }); - } catch (err) { - if (SavedObjectsErrorHelpers.isNotFoundError(err)) { - return null; - } else { - throw err; - } - } - } else { - return null; - } - }; - - /** - * This creates an agnostic space endpoint list if it does not exist. This tries to be - * as fast as possible by ignoring conflict errors and not returning the contents of the - * list if it already exists. - * @returns AssistantAnonymizationFieldSchema if it created the endpoint list, otherwise null if it already exists - */ - public createAnonymizationFields = async ( - items: AnonymizationFieldCreateProps[] - ): Promise => { - const { savedObjectsClient, user } = this; - - const dateNow = new Date().toISOString(); - try { - const formattedItems = items.map((item) => { - return { - attributes: { - created_at: dateNow, - created_by: user, - field_id: item.fieldId, - default_allow: item.defaultAllow ?? false, - default_allow_replacement: item.defaultAllowReplacement ?? false, - updated_by: user, - updated_at: dateNow, - }, - type: assistantAnonymizationFieldsTypeName, - }; - }); - const savedObjectsBulk = - await savedObjectsClient.bulkCreate(formattedItems); - - const result = savedObjectsBulk.saved_objects.map((savedObject) => - transformSavedObjectToAssistantAnonymizationField({ savedObject }) - ); - return result; - } catch (err) { - if (SavedObjectsErrorHelpers.isConflictError(err)) { - return []; - } else { - throw err; - } - } - }; - - /** - * This is the same as "updateExceptionListItem" except it applies specifically to the endpoint list and will - * auto-call the "createEndpointList" for you so that you have the best chance of the endpoint - * being there if it did not exist before. If the list did not exist before, then creating it here will still cause a - * return of null but at least the list exists again. - * @param options - * @param options._version The version to update the endpoint list item to - * @param options.comments The comments of the endpoint list item - * @param options.description The description of the endpoint list item - * @param options.entries The entries of the endpoint list item - * @param options.id The id of the list item (Either this or itemId has to be defined) - * @param options.itemId The item id of the list item (Either this or id has to be defined) - * @param options.meta Optional meta data of the list item - * @param options.name The name of the list item - * @param options.osTypes The OS type of the list item - * @param options.tags Tags of the endpoint list item - * @param options.type The type of the endpoint list item (Default is "simple") - * @returns The exception list item updated, otherwise null if not updated - */ - public updateAnonymizationFields = async ( - items: AnonymizationFieldUpdateProps[] - ): Promise => { - const { savedObjectsClient, user } = this; - const dateNow = new Date().toISOString(); - - const existingItems = ( - await this.findAnonymizationFields({ - page: 1, - perPage: 1000, - filter: items.map((updated) => `id:${updated.id}`).join(' OR '), - fields: ['id'], - }) - ).data.reduce((res, item) => { - res[item.id] = item; - return res; - }, {} as Record); - const formattedItems = items.map((item) => { - return { - attributes: { - default_allow: item.defaultAllow ?? false, - default_allow_replacement: item.defaultAllowReplacement ?? false, - updated_by: user, - updated_at: dateNow, - }, - id: existingItems[item.id].id, - type: assistantAnonymizationFieldsTypeName, - }; - }); - const savedObjectsBulk = - await savedObjectsClient.bulkUpdate(formattedItems); - const result = savedObjectsBulk.saved_objects.map((savedObject) => - transformSavedObjectUpdateToAssistantAnonymizationField({ savedObject }) - ); - return result; - }; - - /** - * Delete the anonymization field by id - * @param options - * @param options.id the "id" of the anonymization field - */ - public deleteAnonymizationFieldsByIds = async ( - ids: string[] - ): Promise => { - const { savedObjectsClient } = this; - - const res = await savedObjectsClient.bulkDelete( - ids.map((id) => ({ id, type: assistantAnonymizationFieldsTypeName })) - ); - return res.statuses; - }; - - /** - * Finds anonymization fields given a set of criteria. - * @param options - * @param options.filter The filter to apply in the search - * @param options.perPage How many per page to return - * @param options.page The page number or "undefined" if there is no page number to continue from - * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in - * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in - * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in - * @returns The found anonymization fields or null if nothing is found - */ - public findAnonymizationFields = async ({ - perPage, - page, - sortField, - sortOrder, - filter, - fields, - }: { - perPage: number; - page: number; - sortField?: string; - sortOrder?: SortOrder; - filter?: string; - fields?: string[]; - }): Promise => { - const { savedObjectsClient } = this; - - const savedObjectsFindResponse = - await savedObjectsClient.find({ - filter, - page, - perPage, - sortField, - sortOrder, - type: assistantAnonymizationFieldsTypeName, - fields, - }); - - return transformSavedObjectsToFoundAssistantAnonymizationField({ savedObjectsFindResponse }); - }; -} diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts b/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts deleted file mode 100644 index 0820fba23e7ba..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/ai_assistant_prompts_so_client.ts +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - Logger, - SavedObjectsErrorHelpers, - type SavedObjectsClientContract, -} from '@kbn/core/server'; - -import { - PromptCreateProps, - PromptResponse, - PromptUpdateProps, - SortOrder, -} from '@kbn/elastic-assistant-common'; -import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; -import { - AssistantPromptSoSchema, - assistantPromptsTypeName, - transformSavedObjectToAssistantPrompt, - transformSavedObjectUpdateToAssistantPrompt, - transformSavedObjectsToFoundAssistantPrompt, -} from './elastic_assistant_prompts_type'; - -export interface ConstructorOptions { - /** User creating, modifying, deleting, or updating the prompts */ - user: string; - /** Saved objects client to create, modify, delete, the prompts */ - savedObjectsClient: SavedObjectsClientContract; - logger: Logger; -} - -/** - * Class for use for prompts that are used for AI assistant. - */ -export class AIAssistantPromptsSOClient { - /** User creating, modifying, deleting, or updating the prompts */ - private readonly user: string; - - /** Saved objects client to create, modify, delete, the prompts */ - private readonly savedObjectsClient: SavedObjectsClientContract; - - /** - * Constructs the assistant client - * @param options - * @param options.user The user associated with the action for exception list - * @param options.savedObjectsClient The saved objects client to create, modify, delete, an AI prompts - */ - constructor({ user, savedObjectsClient }: ConstructorOptions) { - this.user = user; - this.savedObjectsClient = savedObjectsClient; - } - - /** - * Fetch an exception list parent container - * @param options - * @param options.id the "id" of an exception list - * @returns The found exception list or null if none exists - */ - public getPrompt = async (id: string): Promise => { - const { savedObjectsClient } = this; - if (id != null) { - try { - const savedObject = await savedObjectsClient.get( - assistantPromptsTypeName, - id - ); - return transformSavedObjectToAssistantPrompt({ savedObject }); - } catch (err) { - if (SavedObjectsErrorHelpers.isNotFoundError(err)) { - return null; - } else { - throw err; - } - } - } else { - return null; - } - }; - - /** - * This creates an agnostic space endpoint list if it does not exist. This tries to be - * as fast as possible by ignoring conflict errors and not returning the contents of the - * list if it already exists. - * @returns AssistantPromptSchema if it created the endpoint list, otherwise null if it already exists - */ - public createPrompt = async ({ - promptType, - content, - name, - isDefault, - isNewConversationDefault, - }: PromptCreateProps): Promise => { - const { savedObjectsClient, user } = this; - - const dateNow = new Date().toISOString(); - try { - const savedObject = await savedObjectsClient.create( - assistantPromptsTypeName, - { - created_at: dateNow, - created_by: user, - content, - name, - is_default: isDefault ?? false, - is_new_conversation_default: isNewConversationDefault ?? false, - prompt_type: promptType, - updated_by: user, - updated_at: dateNow, - } - ); - return transformSavedObjectToAssistantPrompt({ savedObject }); - } catch (err) { - if (SavedObjectsErrorHelpers.isConflictError(err)) { - return null; - } else { - throw err; - } - } - }; - - /** - * This is the same as "updateExceptionListItem" except it applies specifically to the endpoint list and will - * auto-call the "createEndpointList" for you so that you have the best chance of the endpoint - * being there if it did not exist before. If the list did not exist before, then creating it here will still cause a - * return of null but at least the list exists again. - * @param options - * @param options._version The version to update the endpoint list item to - * @param options.comments The comments of the endpoint list item - * @param options.description The description of the endpoint list item - * @param options.entries The entries of the endpoint list item - * @param options.id The id of the list item (Either this or itemId has to be defined) - * @param options.itemId The item id of the list item (Either this or id has to be defined) - * @param options.meta Optional meta data of the list item - * @param options.name The name of the list item - * @param options.osTypes The OS type of the list item - * @param options.tags Tags of the endpoint list item - * @param options.type The type of the endpoint list item (Default is "simple") - * @returns The exception list item updated, otherwise null if not updated - */ - public updatePromptItem = async ( - prompt: PromptResponse, - { promptType, content, name, isNewConversationDefault }: PromptUpdateProps - ): Promise => { - const { savedObjectsClient, user } = this; - const savedObject = await savedObjectsClient.update( - assistantPromptsTypeName, - prompt.id, - { - content, - is_new_conversation_default: isNewConversationDefault, - prompt_type: promptType, - name, - updated_by: user, - } - ); - return transformSavedObjectUpdateToAssistantPrompt({ - prompt, - savedObject, - }); - }; - - /** - * Delete the prompt by id - * @param options - * @param options.id the "id" of the prompt - */ - public deletePromptById = async (id: string): Promise => { - const { savedObjectsClient } = this; - - await savedObjectsClient.delete(assistantPromptsTypeName, id); - }; - - /** - * Finds prompts given a set of criteria. - * @param options - * @param options.filter The filter to apply in the search - * @param options.perPage How many per page to return - * @param options.page The page number or "undefined" if there is no page number to continue from - * @param options.pit The Point in Time (pit) id if there is one, otherwise "undefined" can be sent in - * @param options.sortField The sort field string if there is one, otherwise "undefined" can be sent in - * @param options.sortOrder The sort order of "asc" or "desc", otherwise "undefined" can be sent in - * @returns The found prompts or null if nothing is found - */ - public findPrompts = async ({ - perPage, - page, - sortField, - sortOrder, - filter, - fields, - }: { - perPage: number; - page: number; - sortField?: string; - sortOrder?: SortOrder; - filter?: string; - fields?: string[]; - }): Promise => { - const { savedObjectsClient } = this; - - const savedObjectsFindResponse = await savedObjectsClient.find({ - filter, - page, - perPage, - sortField, - sortOrder, - type: assistantPromptsTypeName, - fields, - }); - - return transformSavedObjectsToFoundAssistantPrompt({ savedObjectsFindResponse }); - }; -} diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts deleted file mode 100644 index a314e9ab83231..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_anonymization_fields_type.ts +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; -import type { - SavedObject, - SavedObjectsFindResponse, - SavedObjectsType, - SavedObjectsUpdateResponse, -} from '@kbn/core/server'; -import { AnonymizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; -import { FindAnonymizationFieldsResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/find_anonymization_fields_route.gen'; - -export const assistantAnonymizationFieldsTypeName = 'elastic-ai-assistant-anonymization-fields'; - -export const assistantAnonymizationFieldsTypeMappings: SavedObjectsType['mappings'] = { - properties: { - id: { - type: 'keyword', - }, - field_id: { - type: 'keyword', - }, - default_allow: { - type: 'boolean', - }, - default_allow_replacement: { - type: 'boolean', - }, - updated_at: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - created_at: { - type: 'keyword', - }, - created_by: { - type: 'keyword', - }, - }, -}; - -export const transformSavedObjectToAssistantAnonymizationField = ({ - savedObject, -}: { - savedObject: SavedObject; -}): AnonymizationFieldResponse => { - const { - version: _version, - attributes: { - /* eslint-disable @typescript-eslint/naming-convention */ - created_at, - created_by, - field_id, - default_allow, - default_allow_replacement, - updated_by, - updated_at, - /* eslint-enable @typescript-eslint/naming-convention */ - }, - id, - } = savedObject; - - return { - createdAt: created_at, - createdBy: created_by, - fieldId: field_id, - defaultAllow: default_allow, - defaultAllowReplacement: default_allow_replacement, - updatedAt: updated_at, - updatedBy: updated_by, - id, - }; -}; - -export interface AssistantAnonymizationFieldSoSchema { - created_at: string; - created_by: string; - field_id: string; - default_allow?: boolean; - default_allow_replacement?: boolean; - updated_at: string; - updated_by: string; -} - -export const assistantAnonymizationFieldsType: SavedObjectsType = { - name: assistantAnonymizationFieldsTypeName, - indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, // todo: generic - hidden: false, - namespaceType: 'multiple-isolated', - mappings: assistantAnonymizationFieldsTypeMappings, -}; - -export const transformSavedObjectUpdateToAssistantAnonymizationField = ({ - savedObject, -}: { - savedObject: SavedObjectsUpdateResponse; -}): AnonymizationFieldResponse => { - const dateNow = new Date().toISOString(); - const { - version: _version, - attributes: { - updated_by: updatedBy, - default_allow: defaultAllow, - default_allow_replacement: defaultAllowReplacement, - created_at: createdAt, - created_by: createdBy, - field_id: fieldId, - }, - id, - updated_at: updatedAt, - } = savedObject; - - return { - createdAt, - createdBy, - fieldId: fieldId ?? '', - id, - defaultAllow, - defaultAllowReplacement, - updatedAt: updatedAt ?? dateNow, - updatedBy, - }; -}; - -export const transformSavedObjectsToFoundAssistantAnonymizationField = ({ - savedObjectsFindResponse, -}: { - savedObjectsFindResponse: SavedObjectsFindResponse; -}): FindAnonymizationFieldsResponse => { - return { - data: savedObjectsFindResponse.saved_objects.map((savedObject) => - transformSavedObjectToAssistantAnonymizationField({ savedObject }) - ), - page: savedObjectsFindResponse.page, - perPage: savedObjectsFindResponse.per_page, - total: savedObjectsFindResponse.total, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts b/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts deleted file mode 100644 index 2e1c67a02b936..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/elastic_assistant_prompts_type.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; -import type { - SavedObject, - SavedObjectsFindResponse, - SavedObjectsType, - SavedObjectsUpdateResponse, -} from '@kbn/core/server'; -import { PromptResponse } from '@kbn/elastic-assistant-common'; -import { FindPromptsResponse } from '@kbn/elastic-assistant-common/impl/schemas/prompts/find_prompts_route.gen'; - -export const assistantPromptsTypeName = 'elastic-ai-assistant-prompts'; - -export const assistantPromptsTypeMappings: SavedObjectsType['mappings'] = { - properties: { - id: { - type: 'keyword', - }, - is_default: { - type: 'boolean', - }, - is_shared: { - type: 'boolean', - }, - is_new_conversation_default: { - type: 'boolean', - }, - name: { - type: 'keyword', - }, - prompt_type: { - type: 'keyword', - }, - content: { - type: 'keyword', - }, - updated_at: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - created_at: { - type: 'keyword', - }, - created_by: { - type: 'keyword', - }, - }, -}; - -export const assistantPromptsType: SavedObjectsType = { - name: assistantPromptsTypeName, - indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, - hidden: false, - namespaceType: 'multiple-isolated', - mappings: assistantPromptsTypeMappings, -}; - -export interface AssistantPromptSoSchema { - created_at: string; - created_by: string; - content: string; - is_default?: boolean; - is_shared?: boolean; - is_new_conversation_default?: boolean; - name: string; - prompt_type: string; - updated_at: string; - updated_by: string; -} - -export const transformSavedObjectToAssistantPrompt = ({ - savedObject, -}: { - savedObject: SavedObject; -}): PromptResponse => { - const { - version: _version, - attributes: { - /* eslint-disable @typescript-eslint/naming-convention */ - created_at, - created_by, - content, - is_default, - is_new_conversation_default, - prompt_type, - name, - updated_by, - updated_at, - /* eslint-enable @typescript-eslint/naming-convention */ - }, - id, - } = savedObject; - - return { - createdAt: created_at, - createdBy: created_by, - content, - name, - promptType: prompt_type, - isDefault: is_default, - isNewConversationDefault: is_new_conversation_default, - updatedAt: updated_at, - updatedBy: updated_by, - id, - }; -}; - -export const transformSavedObjectUpdateToAssistantPrompt = ({ - prompt, - savedObject, -}: { - prompt: PromptResponse; - savedObject: SavedObjectsUpdateResponse; -}): PromptResponse => { - const dateNow = new Date().toISOString(); - const { - version: _version, - attributes: { - name, - updated_by: updatedBy, - content, - prompt_type: promptType, - is_new_conversation_default: isNewConversationDefault, - }, - id, - updated_at: updatedAt, - } = savedObject; - - return { - createdAt: prompt.createdAt, - createdBy: prompt.createdBy, - content: content ?? prompt.content, - promptType: promptType ?? prompt.promptType, - id, - isDefault: prompt.isDefault, - isNewConversationDefault: isNewConversationDefault ?? prompt.isNewConversationDefault, - name: name ?? prompt.name, - updatedAt: updatedAt ?? dateNow, - updatedBy: updatedBy ?? prompt.updatedBy, - }; -}; - -export const transformSavedObjectsToFoundAssistantPrompt = ({ - savedObjectsFindResponse, -}: { - savedObjectsFindResponse: SavedObjectsFindResponse; -}): FindPromptsResponse => { - return { - data: savedObjectsFindResponse.saved_objects.map((savedObject) => - transformSavedObjectToAssistantPrompt({ savedObject }) - ), - page: savedObjectsFindResponse.page, - perPage: savedObjectsFindResponse.per_page, - total: savedObjectsFindResponse.total, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/saved_object/index.ts b/x-pack/plugins/elastic_assistant/server/saved_object/index.ts deleted file mode 100644 index 9d2ae737a4b07..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/saved_object/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './elastic_assistant_prompts_type'; -export * from './elastic_assistant_anonymization_fields_type'; diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index fe97c87539316..f4beaeeace391 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -28,9 +28,9 @@ import { RetrievalQAChain } from 'langchain/chains'; import { ElasticsearchClient } from '@kbn/core/server'; import { AssistantFeatures, ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common'; import { AIAssistantConversationsDataClient } from './conversations_data_client'; -import { AIAssistantPromptsSOClient } from './saved_object/ai_assistant_prompts_so_client'; +import { AIAssistantPromtsDataClient } from './promts_data_client'; +import { AIAssistantAnonymizationFieldsDataClient } from './anonymization_fields_data_client'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; -import { AIAssistantAnonymizationFieldsSOClient } from './saved_object/ai_assistant_anonymization_fields_so_client'; export const PLUGIN_ID = 'elasticAssistant' as const; @@ -99,8 +99,8 @@ export interface ElasticAssistantApiRequestHandlerContext { getSpaceId: () => string; getCurrentUser: () => AuthenticatedUser | null; getAIAssistantConversationsDataClient: () => Promise; - getAIAssistantPromptsSOClient: () => AIAssistantPromptsSOClient; - getAIAssistantAnonymizationFieldsSOClient: () => AIAssistantAnonymizationFieldsSOClient; + getAIAssistantPromptsDataClient: () => Promise; + getAIAssistantAnonymizationFieldsDataClient: () => Promise; telemetry: AnalyticsServiceSetup; } /** @@ -132,18 +132,26 @@ export interface InitAssistantResult { export interface AssistantResourceNames { componentTemplate: { conversations: string; + prompts: string; + anonymizationFields: string; kb: string; }; indexTemplate: { conversations: string; + prompts: string; + anonymizationFields: string; kb: string; }; aliases: { conversations: string; + prompts: string; + anonymizationFields: string; kb: string; }; indexPatterns: { conversations: string; + prompts: string; + anonymizationFields: string; kb: string; }; pipelines: { From bf82ec1376d2de1ff829073eb43f8d50f191d1af Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 14:44:12 -0800 Subject: [PATCH 081/141] - --- packages/kbn-check-mappings-update-cli/current_mappings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 3c7a34a02a956..aaf612ed8ed56 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3243,4 +3243,4 @@ "dynamic": false, "properties": {} } -} \ No newline at end of file +} From 94836be4f4da4b56e5abb182760fee19c44c3bf6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 13 Feb 2024 22:52:23 +0000 Subject: [PATCH 082/141] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/elastic_assistant/tsconfig.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 2dc7998244cd9..22faa17c770ce 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -35,7 +35,6 @@ "@kbn/elastic-assistant-common", "@kbn/core-http-router-server-mocks", "@kbn/data-stream-adapter", - "@kbn/alerts-as-data-utils", "@kbn/core-saved-objects-utils-server", "@kbn/core-elasticsearch-client-server-mocks", "@kbn/task-manager-plugin", @@ -44,7 +43,6 @@ "@kbn/es-types", "@kbn/config-schema", "@kbn/zod-helpers", - "@kbn/core-saved-objects-server", "@kbn/spaces-plugin", "@kbn/zod-helpers", "@kbn/security-plugin-types-common", From 6439c012f1be926f0bb3325eee1b90784d311b1e Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 17:14:44 -0800 Subject: [PATCH 083/141] added sendMessage telemetry to user_chat_send --- ...ost_actions_connector_execute_route.gen.ts | 2 +- ...ctions_connector_execute_route.schema.yaml | 1 - .../api/conversations/conversations.ts | 41 ---- .../chat_send/use_chat_send.test.tsx | 16 +- .../assistant/chat_send/use_chat_send.tsx | 22 ++- .../conversation_selector_settings/index.tsx | 4 +- .../conversation_settings.test.tsx | 3 + .../assistant/use_conversation/index.test.tsx | 72 ------- .../impl/assistant/use_conversation/index.tsx | 99 ++-------- .../connectorland/connector_setup/index.tsx | 20 +- .../elastic_assistant/server/lib/executor.ts | 6 +- .../server/lib/parse_stream.ts | 10 +- .../server/routes/evaluate/post_evaluate.ts | 2 + .../routes/post_actions_connector_execute.ts | 181 ++++++++++-------- .../server/routes/request_context_factory.ts | 3 + 15 files changed, 163 insertions(+), 319 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts index 97fd0a663a79f..e848e7f84d046 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen.ts @@ -55,7 +55,7 @@ export type ExecuteConnectorRequestParamsInput = z.input; export const ExecuteConnectorRequestBody = z.object({ - conversationId: UUID, + conversationId: UUID.optional(), params: ConnectorExecutionParams, alertsIndexPattern: z.string().optional(), allow: z.array(z.string()).optional(), diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml index f50074e322320..5810e600c0f52 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.schema.yaml @@ -26,7 +26,6 @@ paths: type: object required: - params - - conversationId - llmType properties: conversationId: diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts index 3589523b4ac6d..6001d51cada0a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/conversations.ts @@ -214,44 +214,3 @@ export const updateConversation = async ({ }); } }; - -/** - * API call for evaluating models. - * - * @param {PutConversationMessageParams} options - The options object. - * - * @returns {Promise} - */ -export const appendConversationMessages = async ({ - http, - conversationId, - messages, - signal, - toasts, -}: PutConversationMessageParams): Promise => { - try { - const response = await http.fetch( - `${ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL}/${conversationId}/messages`, - { - method: 'POST', - body: JSON.stringify({ - messages, - }), - headers: { - 'Content-Type': 'application/json', - }, - version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, - signal, - } - ); - - return response as Conversation; - } catch (error) { - toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, { - title: i18n.translate('xpack.elasticAssistant.conversations.updateConversationError', { - defaultMessage: 'Error updating conversation by id {conversationId}', - values: { conversationId }, - }), - }); - } -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index b865200017eb8..c91e2f2fa10a1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -23,9 +23,7 @@ const setPromptTextPreview = jest.fn(); const setSelectedPromptContexts = jest.fn(); const setUserPrompt = jest.fn(); const sendMessages = jest.fn(); -const appendMessage = jest.fn(); const removeLastMessage = jest.fn(); -const appendReplacements = jest.fn(); const clearConversation = jest.fn(); const refresh = jest.fn(); const setCurrentConversation = jest.fn(); @@ -59,13 +57,11 @@ describe('use chat send', () => { sendMessages: sendMessages.mockReturnValue(robotMessage), }); (useConversation as jest.Mock).mockReturnValue({ - appendMessage, - appendReplacements, removeLastMessage, clearConversation, }); }); - it('handleOnChatCleared clears the conversation', () => { + it('handleOnChatCleared clears the conversation', async () => { const { result } = renderHook(() => useChatSend(testProps), { wrapper: TestProviders, }); @@ -74,7 +70,10 @@ describe('use chat send', () => { expect(setPromptTextPreview).toHaveBeenCalledWith(''); expect(setUserPrompt).toHaveBeenCalledWith(''); expect(setSelectedPromptContexts).toHaveBeenCalledWith({}); - expect(clearConversation).toHaveBeenCalledWith(testProps.currentConversation.id); + await waitFor(() => { + expect(clearConversation).toHaveBeenCalledWith(testProps.currentConversation.id); + expect(refresh).toHaveBeenCalled(); + }); expect(setEditingSystemPromptId).toHaveBeenCalledWith(defaultSystemPrompt.id); }); it('handlePromptChange updates prompt successfully', () => { @@ -95,14 +94,11 @@ describe('use chat send', () => { await waitFor(() => { expect(sendMessages).toHaveBeenCalled(); - const appendMessageSend = setCurrentConversation.mock.calls[1][0].messages[0]; - const appendMessageResponse = setCurrentConversation.mock.calls[1][0].messages[1]; + const appendMessageSend = sendMessages.mock.calls[0][0].messages[0]; expect(appendMessageSend.content).toEqual( `You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL or EQL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\nUse the following context to answer questions:\n\n\n\n${promptText}` ); expect(appendMessageSend.role).toEqual('user'); - expect(appendMessageResponse.content).toEqual(robotMessage.response); - expect(appendMessageResponse.role).toEqual('assistant'); }); }); it('handleButtonSendMessage sends message with only provided prompt text and context already exists in convo history', async () => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index e8a2b95fb6acb..5ab82d8ab511d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -12,7 +12,7 @@ import { SelectedPromptContext } from '../prompt_context/types'; import { useSendMessages } from '../use_send_messages'; import { useConversation } from '../use_conversation'; import { getCombinedMessage } from '../prompt/helpers'; -import { Conversation, Message, Prompt } from '../../..'; +import { Conversation, Message, Prompt, useAssistantContext } from '../../..'; import { getMessageFromRawResponse } from '../helpers'; import { getDefaultSystemPrompt } from '../use_conversation/helpers'; @@ -59,6 +59,11 @@ export const useChatSend = ({ refresh, setCurrentConversation, }: UseChatSendProps): UseChatSend => { + const { + assistantTelemetry, + knowledgeBase: { isEnabledKnowledgeBase, isEnabledRAGAlerts }, + } = useAssistantContext(); + const { isLoading, sendMessages } = useSendMessages(); const { clearConversation, removeLastMessage } = useConversation(); @@ -113,6 +118,12 @@ export const useChatSend = ({ conversationId: currentConversation.id, replacements, }); + assistantTelemetry?.reportAssistantMessageSent({ + conversationId: currentConversation.title, + role: userMessage.role, + isEnabledKnowledgeBase, + isEnabledRAGAlerts, + }); const responseMessage: Message = getMessageFromRawResponse(rawResponse); @@ -120,12 +131,21 @@ export const useChatSend = ({ ...currentConversation, messages: [...updatedMessages, responseMessage], }); + assistantTelemetry?.reportAssistantMessageSent({ + conversationId: currentConversation.title, + role: responseMessage.role, + isEnabledKnowledgeBase, + isEnabledRAGAlerts, + }); }, [ allSystemPrompts, + assistantTelemetry, currentConversation, editingSystemPromptId, http, + isEnabledKnowledgeBase, + isEnabledRAGAlerts, selectedPromptContexts, sendMessages, setCurrentConversation, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx index 2306b74ebca4d..17aed40340972 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector_settings/index.tsx @@ -168,8 +168,8 @@ export const ConversationSelectorSettings: React.FC = React.memo( option: ConversationSelectorSettingsOption, searchValue: string, OPTION_CONTENT_CLASSNAME: string - ) => React.ReactNode = (option, searchValue, contentClassName) => { - const { label, value, id } = option; + ) => React.ReactNode = (option, searchValue) => { + const { label, value } = option; return ( { ...welcomeConvo, apiConfig: { connectorId: mockConnector.id, + connectorTypeTitle: 'OpenAI', + model: undefined, + provider: undefined, }, }, }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index 697f841acaf0f..d117579443146 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -8,16 +8,13 @@ import { useConversation } from '.'; import { act, renderHook } from '@testing-library/react-hooks'; import { TestProviders } from '../../mock/test_providers/test_providers'; -import { welcomeConvo } from '../../mock/conversation'; import React from 'react'; import { ConversationRole } from '../../assistant_context/types'; import { httpServiceMock } from '@kbn/core/public/mocks'; import { WELCOME_CONVERSATION } from './sample_conversations'; import { - appendConversationMessages as _appendConversationMessagesApi, deleteConversation, getConversationById as _getConversationById, - updateConversation, createConversation as _createConversationApi, } from '../api/conversations'; @@ -40,7 +37,6 @@ const mockConvo = { apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, }; -const appendConversationMessagesApi = _appendConversationMessagesApi as jest.Mock; const getConversationById = _getConversationById as jest.Mock; const createConversation = _createConversationApi as jest.Mock; @@ -53,42 +49,6 @@ describe('useConversation', () => { jest.clearAllMocks(); }); - it('should report telemetry when a message has been sent', async () => { - await act(async () => { - const reportAssistantMessageSent = jest.fn(); - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - {}, - reportAssistantQuickPrompt: () => {}, - reportAssistantSettingToggled: () => {}, - reportAssistantMessageSent, - }, - }} - > - {children} - - ), - }); - await waitForNextUpdate(); - - appendConversationMessagesApi.mockResolvedValue([message, anotherMessage, message]); - await result.current.appendMessage({ - id: 'longuuid', - title: welcomeConvo.id, - message, - }); - expect(reportAssistantMessageSent).toHaveBeenCalledWith({ - conversationId: 'Welcome', - isEnabledKnowledgeBase: false, - isEnabledRAGAlerts: false, - role: 'user', - }); - }); - }); - it('should create a new conversation when called with valid conversationId and message', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { @@ -149,38 +109,6 @@ describe('useConversation', () => { }); }); - it('appends replacements', async () => { - await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useConversation(), { - wrapper: ({ children }) => ( - {children} - ), - }); - await waitForNextUpdate(); - - getConversationById.mockResolvedValue(mockConvo); - - await result.current.appendReplacements({ - conversationId: welcomeConvo.id, - replacements: { - '1.0.0.721': '127.0.0.1', - '1.0.0.01': '10.0.0.1', - 'tsoh-tset': 'test-host', - }, - }); - - expect(updateConversation).toHaveBeenCalledWith({ - http: httpMock, - conversationId: welcomeConvo.id, - replacements: { - '1.0.0.721': '127.0.0.1', - '1.0.0.01': '10.0.0.1', - 'tsoh-tset': 'test-host', - }, - }); - }); - }); - it('should remove the last message from a conversation when called with valid conversationId', async () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useConversation(), { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 81798ad832714..ed26f661e3a4d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -12,7 +12,6 @@ import { Conversation, Message } from '../../assistant_context/types'; import * as i18n from './translations'; import { getDefaultSystemPrompt } from './helpers'; import { - appendConversationMessages, createConversation as createConversationApi, deleteConversation as deleteConversationApi, getConversationById, @@ -27,17 +26,6 @@ export const DEFAULT_CONVERSATION_STATE: Conversation = { title: i18n.DEFAULT_CONVERSATION_TITLE, }; -interface AppendMessageProps { - id: string; - title: string; - message: Message; -} - -interface AppendReplacementsProps { - conversationId: string; - replacements: Record; -} - interface CreateConversationProps { conversationId: string; messages?: Message[]; @@ -49,11 +37,6 @@ interface SetApiConfigProps { } interface UseConversation { - appendMessage: ({ id, title, message }: AppendMessageProps) => Promise; - appendReplacements: ({ - conversationId, - replacements, - }: AppendReplacementsProps) => Promise | undefined>; clearConversation: (conversationId: string) => Promise; getDefaultConversation: ({ conversationId, messages }: CreateConversationProps) => Conversation; deleteConversation: (conversationId: string) => void; @@ -67,18 +50,13 @@ interface UseConversation { } export const useConversation = (): UseConversation => { - const { - allSystemPrompts, - assistantTelemetry, - knowledgeBase: { isEnabledKnowledgeBase, isEnabledRAGAlerts }, - http, - } = useAssistantContext(); + const { allSystemPrompts, http, toasts } = useAssistantContext(); const getConversation = useCallback( async (conversationId: string) => { - return getConversationById({ http, id: conversationId }); + return getConversationById({ http, id: conversationId, toasts }); }, - [http] + [http, toasts] ); /** @@ -87,70 +65,24 @@ export const useConversation = (): UseConversation => { const removeLastMessage = useCallback( async (conversationId: string) => { let messages: Message[] = []; - const prevConversation = await getConversationById({ http, id: conversationId }); + const prevConversation = await getConversationById({ http, id: conversationId, toasts }); if (prevConversation != null) { messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); await updateConversation({ http, conversationId, messages, + toasts, }); } return messages; }, - [http] - ); - - /** - * Append a message to the conversation[] for a given conversationId - */ - const appendMessage = useCallback( - async ({ id, message, title }: AppendMessageProps): Promise => { - assistantTelemetry?.reportAssistantMessageSent({ - conversationId: title, - role: message.role, - isEnabledKnowledgeBase, - isEnabledRAGAlerts, - }); - - const res = await appendConversationMessages({ - http, - conversationId: id, - messages: [message], - }); - return res?.messages; - }, - [assistantTelemetry, isEnabledKnowledgeBase, isEnabledRAGAlerts, http] - ); - - const appendReplacements = useCallback( - async ({ - conversationId, - replacements, - }: AppendReplacementsProps): Promise | undefined> => { - let allReplacements = replacements; - const prevConversation = await getConversationById({ http, id: conversationId }); - if (prevConversation != null) { - allReplacements = { - ...prevConversation.replacements, - ...replacements, - }; - - await updateConversation({ - http, - conversationId, - replacements: allReplacements, - }); - } - - return allReplacements; - }, - [http] + [http, toasts] ); const clearConversation = useCallback( async (conversationId: string) => { - const prevConversation = await getConversationById({ http, id: conversationId }); + const prevConversation = await getConversationById({ http, id: conversationId, toasts }); if (prevConversation) { const defaultSystemPromptId = getDefaultSystemPrompt({ allSystemPrompts, @@ -159,6 +91,7 @@ export const useConversation = (): UseConversation => { await updateConversation({ http, + toasts, conversationId, apiConfig: { defaultSystemPromptId, @@ -168,7 +101,7 @@ export const useConversation = (): UseConversation => { }); } }, - [allSystemPrompts, http] + [allSystemPrompts, http, toasts] ); /** @@ -204,9 +137,9 @@ export const useConversation = (): UseConversation => { */ const createConversation = useCallback( async (conversation: Conversation): Promise => { - return createConversationApi({ http, conversation }); + return createConversationApi({ http, conversation, toasts }); }, - [http] + [http, toasts] ); /** @@ -214,9 +147,9 @@ export const useConversation = (): UseConversation => { */ const deleteConversation = useCallback( async (conversationId: string): Promise => { - await deleteConversationApi({ http, id: conversationId }); + await deleteConversationApi({ http, id: conversationId, toasts }); }, - [http] + [http, toasts] ); /** @@ -236,21 +169,21 @@ export const useConversation = (): UseConversation => { id: '', messages: conversation.messages ?? [], }, + toasts, }); } else { return updateConversation({ http, conversationId: conversation.id, apiConfig, + toasts, }); } }, - [http] + [http, toasts] ); return { - appendMessage, - appendReplacements, clearConversation, getDefaultConversation, deleteConversation, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index 7d7a2c97c9c12..4f200bd5b593f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -49,7 +49,7 @@ export const useConnectorSetup = ({ comments: EuiCommentProps[]; prompt: React.ReactElement; } => { - const { appendMessage, setApiConfig } = useConversation(); + const { setApiConfig } = useConversation(); const bottomRef = useRef(null); // Access all conversations so we can add connector to all on initial setup const { actionTypeRegistry, http } = useAssistantContext(); @@ -193,25 +193,9 @@ export const useConnectorSetup = ({ refetchConnectors?.(); setIsConnectorModalVisible(false); - await appendMessage({ - id: updatedConversation.id, - title: updatedConversation.title, - message: { - role: 'assistant', - content: i18n.CONNECTOR_SETUP_COMPLETE, - timestamp: new Date().toLocaleString(), - }, - }); } }, - [ - actionTypeRegistry, - appendMessage, - conversation, - onConversationUpdate, - refetchConnectors, - setApiConfig, - ] + [actionTypeRegistry, conversation, onConversationUpdate, refetchConnectors, setApiConfig] ); return { diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index ed2ce63ce6639..ce7de431ad367 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -16,7 +16,7 @@ import { import { handleStreamStorage } from './parse_stream'; export interface Props { - onMessageSent: (content: string) => void; + onMessageSent?: (content: string) => void; actions: ActionsPluginStart; connectorId: string; params: ConnectorExecutionParams; @@ -49,7 +49,9 @@ export const executeAction = async ({ } const content = get('data.message', actionResult); if (typeof content === 'string') { - onMessageSent(content); + if (onMessageSent) { + onMessageSent(content); + } return { connector_id: connectorId, data: content, // the response from the actions framework diff --git a/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts index 58123e3db8ce0..fed0a1cfb37bd 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/parse_stream.ts @@ -15,15 +15,19 @@ type StreamParser = (responseStream: Readable) => Promise; export const handleStreamStorage: ( responseStream: Readable, llmType: string, - onMessageSent: (content: string) => void + onMessageSent?: (content: string) => void ) => Promise = async (responseStream, llmType, onMessageSent) => { try { const parser = llmType === 'bedrock' ? parseBedrockStream : parseOpenAIStream; // TODO @steph add abort signal const parsedResponse = await parser(responseStream); - onMessageSent(parsedResponse); + if (onMessageSent) { + onMessageSent(parsedResponse); + } } catch (e) { - onMessageSent(`An error occurred while streaming the response:\n\n${e.message}`); + if (onMessageSent) { + onMessageSent(`An error occurred while streaming the response:\n\n${e.message}`); + } } }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index f43985187e238..d9bfd23680f3d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -158,6 +158,8 @@ export const postEvaluateRoute = ( size: DEFAULT_SIZE, isEnabledKnowledgeBase: true, isEnabledRAGAlerts: true, + conversationId: '', + llmType: 'openai', }, }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index a183774f9fe47..0f2aa5926cb61 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -10,6 +10,7 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { schema } from '@kbn/config-schema'; import { + ConversationResponse, ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, ExecuteConnectorRequestBody, Message, @@ -73,78 +74,79 @@ export const postActionsConnectorExecuteRoute = ( body: `Authenticated user not found`, }); } - const dataClient = await assistantContext.getAIAssistantConversationsDataClient(); - const conversation = await dataClient?.getConversation({ - id: request.body.conversationId, - authenticatedUser, - }); - if (conversation == null) { - return response.notFound({ - body: `conversation id: "${request.body.conversationId}" not found`, - }); - } - - if (request.body.replacements) { - await dataClient?.updateConversation({ - existingConversation: conversation, - conversationUpdateProps: { - id: request.body.conversationId, - replacements: request.body.replacements, - }, + let onMessageSent; + let conversation: ConversationResponse | undefined | null; + let prevMessages; + if (request.body.conversationId) { + conversation = await dataClient?.getConversation({ + id: request.body.conversationId, + authenticatedUser, }); - } + prevMessages = conversation?.messages?.map((c) => ({ + role: c.role, + content: c.content, + })); - const dateTimeString = new Date().toLocaleString(); + if (conversation == null) { + return response.notFound({ + body: `conversation id: "${request.body.conversationId}" not found`, + }); + } - const appendMessageFuncs = request.body.params.subActionParams.messages.map( - (userMessage) => async () => { - const res = await dataClient?.appendConversationMessages({ + if (request.body.replacements) { + await dataClient?.updateConversation({ existingConversation: conversation, - messages: request.body.params.subActionParams.messages.map((m) => ({ - content: getMessageContentWithoutReplacements({ - messageContent: userMessage.content, - replacements: request.body.replacements as Record | undefined, - }), - role: m.role, - timestamp: dateTimeString, - })), + conversationUpdateProps: { + id: request.body.conversationId, + replacements: request.body.replacements, + }, }); - if (res == null) { - return response.badRequest({ - body: `conversation id: "${request.body.conversationId}" not updated`, - }); - } } - ); - await Promise.all(appendMessageFuncs.map((appendMessageFunc) => appendMessageFunc())); + const dateTimeString = new Date().toLocaleString(); - const updatedConversation = await dataClient?.getConversation({ - id: request.body.conversationId, - authenticatedUser, - }); + const appendMessageFuncs = request.body.params.subActionParams.messages.map( + (userMessage) => async () => { + if (conversation != null) { + const res = await dataClient?.appendConversationMessages({ + existingConversation: conversation, + messages: request.body.params.subActionParams.messages.map((m) => ({ + content: getMessageContentWithoutReplacements({ + messageContent: userMessage.content, + replacements: request.body.replacements as + | Record + | undefined, + }), + role: m.role, + timestamp: dateTimeString, + })), + }); + if (res == null) { + return response.badRequest({ + body: `conversation id: "${request.body.conversationId}" not updated`, + }); + } + } + } + ); - if (updatedConversation == null) { - return response.notFound({ - body: `conversation id: "${request.body.conversationId}" not found`, - }); - } + await Promise.all(appendMessageFuncs.map((appendMessageFunc) => appendMessageFunc())); - const connectorId = decodeURIComponent(request.params.connectorId); + const updatedConversation = await dataClient?.getConversation({ + id: request.body.conversationId, + authenticatedUser, + }); - // get the actions plugin start contract from the request context: - const actions = (await context.elasticAssistant).actions; + if (updatedConversation == null) { + return response.notFound({ + body: `conversation id: "${request.body.conversationId}" not found`, + }); + } - // if not langchain, call execute action directly and return the response: - if ( - !request.body.isEnabledKnowledgeBase && - !requestHasRequiredAnonymizationParams(request) - ) { - logger.debug('Executing via actions framework directly'); - const result = await executeAction({ - onMessageSent: (content) => { + onMessageSent = (content: string) => { + if (updatedConversation) { dataClient?.appendConversationMessages({ existingConversation: updatedConversation, messages: [ @@ -158,7 +160,23 @@ export const postActionsConnectorExecuteRoute = ( }), ], }); - }, + } + }; + } + + const connectorId = decodeURIComponent(request.params.connectorId); + + // get the actions plugin start contract from the request context: + const actions = (await context.elasticAssistant).actions; + + // if not langchain, call execute action directly and return the response: + if ( + !request.body.isEnabledKnowledgeBase && + !requestHasRequiredAnonymizationParams(request) + ) { + logger.debug('Executing via actions framework directly'); + const result = await executeAction({ + onMessageSent, actions, request, connectorId, @@ -167,10 +185,7 @@ export const postActionsConnectorExecuteRoute = ( subActionParams: { ...request.body.params.subActionParams, messages: [ - ...(conversation.messages?.map((c) => ({ - role: c.role, - content: c.content, - })) ?? []), + ...(prevMessages ?? []), ...request.body.params.subActionParams.messages, ], }, @@ -203,13 +218,8 @@ export const postActionsConnectorExecuteRoute = ( // convert the assistant messages to LangChain messages: const langChainMessages = getLangChainMessages( - ([ - ...(conversation.messages?.map((c) => ({ - role: c.role, - content: c.content, - })) ?? []), - request.body.params.subActionParams.messages, - ] ?? []) as unknown as Array> + ([...(prevMessages ?? []), request.body.params.subActionParams.messages] ?? + []) as unknown as Array> ); const elserId = await getElser(request, (await context.core).savedObjects.getClient()); @@ -244,21 +254,22 @@ export const postActionsConnectorExecuteRoute = ( isEnabledRAGAlerts: request.body.isEnabledRAGAlerts, }); - dataClient?.appendConversationMessages({ - existingConversation: conversation, - messages: [ - getMessageFromRawResponse({ - rawContent: langChainResponseBody.data, - traceData: langChainResponseBody.trace_data - ? { - traceId: langChainResponseBody.trace_data.trace_id, - transactionId: langChainResponseBody.trace_data.transaction_id, - } - : {}, - }), - ], - }); - + if (conversation != null) { + dataClient?.appendConversationMessages({ + existingConversation: conversation, + messages: [ + getMessageFromRawResponse({ + rawContent: langChainResponseBody.data, + traceData: langChainResponseBody.trace_data + ? { + traceId: langChainResponseBody.trace_data.trace_id, + transactionId: langChainResponseBody.trace_data.transaction_id, + } + : {}, + }), + ], + }); + } return response.ok({ body: { ...langChainResponseBody, diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 82e21a8cd8690..cb729efb4b902 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -87,6 +87,7 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, + telemetry: core.analytics, }); }), @@ -96,6 +97,7 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, + telemetry: core.analytics, }); }), @@ -105,6 +107,7 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, + telemetry: core.analytics, }); }), }; From 003f8b6ab5964aaedea9eec124d1a2deb3950010 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 17:33:23 -0800 Subject: [PATCH 084/141] fixed tests --- .../__mocks__/conversations_schema.mock.ts | 8 +++-- .../server/ai_assistant_service/index.test.ts | 16 ++++----- .../server/ai_assistant_service/index.ts | 18 ++++++++-- .../get_conversation.test.ts | 20 ++++++----- .../conversations_data_client/index.test.ts | 35 +++++++------------ .../data_client/documents_data_writer.test.ts | 4 +-- .../server/routes/request_context_factory.ts | 3 -- .../routes/user_conversations/create_route.ts | 2 +- .../find_user_conversations_route.ts | 4 +-- 9 files changed, 56 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 7e194487426df..647f466a3d11a 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -86,9 +86,11 @@ export const getConversationMock = ( createdAt: '2019-12-13T16:40:33.400Z', updatedAt: '2019-12-13T16:40:33.400Z', namespace: 'default', - user: { - name: 'elastic', - }, + users: [ + { + name: 'elastic', + }, + ], }); export const getQueryConversationParams = ( diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts index 2aa2f09900fe3..568c6dde3f58c 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.test.ts @@ -119,7 +119,7 @@ describe('AI Assistant Service', () => { ); expect(assistantService.isInitialized()).toEqual(true); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); const componentTemplate = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; expect(componentTemplate.name).toEqual( @@ -616,9 +616,7 @@ describe('AI Assistant Service', () => { mappings: {}, }, })); - clusterClient.indices.putIndexTemplate.mockRejectedValueOnce( - new Error('fail index template') - ); + clusterClient.indices.putIndexTemplate.mockRejectedValue(new Error('fail index template')); assistantService = new AIAssistantService({ logger, @@ -676,7 +674,7 @@ describe('AI Assistant Service', () => { 'AI Assistant service initialized', async () => assistantService.isInitialized() === true ); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); }); test('should retry updating index template for transient ES errors', async () => { @@ -703,7 +701,7 @@ describe('AI Assistant Service', () => { async () => (await getSpaceResourcesInitialized(assistantService)) === true ); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(5); }); test('should retry updating index settings for existing indices for transient ES errors', async () => { @@ -729,7 +727,7 @@ describe('AI Assistant Service', () => { async () => (await getSpaceResourcesInitialized(assistantService)) === true ); - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(5); }); test('should retry updating index mappings for existing indices for transient ES errors', async () => { @@ -755,7 +753,7 @@ describe('AI Assistant Service', () => { async () => (await getSpaceResourcesInitialized(assistantService)) === true ); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(5); }); test('should retry creating concrete index for transient ES errors', async () => { @@ -791,7 +789,7 @@ describe('AI Assistant Service', () => { async () => (await getSpaceResourcesInitialized(assistantService)) === true ); - expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.createDataStream).toHaveBeenCalledTimes(5); }); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 89a3d272d072d..22a959b62ece6 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -225,7 +225,11 @@ export class AIAssistantService { public async createAIAssistantConversationsDataClient( opts: CreateAIAssistantClientParams ): Promise { - await this.checkResourcesInstallation(opts); + const res = await this.checkResourcesInstallation(opts); + + if (res === null) { + return null; + } return new AIAssistantConversationsDataClient({ logger: this.options.logger, @@ -240,7 +244,11 @@ export class AIAssistantService { public async createAIAssistantPromptsDataClient( opts: CreateAIAssistantClientParams ): Promise { - await this.checkResourcesInstallation(opts); + const res = await this.checkResourcesInstallation(opts); + + if (res === null) { + return null; + } return new AIAssistantPromtsDataClient({ logger: this.options.logger, @@ -255,7 +263,11 @@ export class AIAssistantService { public async createAIAssistantAnonymizationFieldsDataClient( opts: CreateAIAssistantClientParams ): Promise { - await this.checkResourcesInstallation(opts); + const res = await this.checkResourcesInstallation(opts); + + if (res === null) { + return null; + } return new AIAssistantAnonymizationFieldsDataClient({ logger: this.options.logger, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index 9aadc63c926ff..68a0db9250600 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -31,10 +31,12 @@ export const getConversationResponseMock = (): ConversationResponse => ({ model: 'test', provider: 'Azure OpenAI', }, - user: { - id: '1111', - name: 'elastic', - }, + users: [ + { + id: '1111', + name: 'elastic', + }, + ], replacements: undefined, }); @@ -78,10 +80,12 @@ export const getSearchConversationMock = model: 'test', provider: 'Azure OpenAI', }, - user: { - id: '1111', - name: 'elastic', - }, + users: [ + { + id: '1111', + name: 'elastic', + }, + ], replacements: undefined, }, }, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts index 5e72b16404043..a406944ff9f8b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/index.test.ts @@ -70,43 +70,36 @@ describe('AIAssistantConversationsDataClient', () => { created_at: '2024-01-25T01:32:37.649Z', messages: [ { - presentation: { - delay: 1000, - stream: true, - }, '@timestamp': '1/24/2024, 5:32:19 PM', role: 'assistant', reader: null, is_error: null, - replacements: null, content: 'Go ahead and click the add connector button below to continue the conversation!', }, { - presentation: null, '@timestamp': '1/24/2024, 5:32:37 PM', role: 'assistant', reader: null, is_error: null, - replacements: null, content: 'Connector setup complete!', }, { - presentation: null, '@timestamp': '1/24/2024, 5:34:50 PM', role: 'assistant', reader: null, is_error: true, - replacements: null, content: 'An error occurred sending your message.', }, ], title: 'Alert summary', is_default: true, - user: { - name: 'elastic', - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - }, + users: [ + { + name: 'elastic', + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + }, + ], }, }, ], @@ -136,10 +129,6 @@ describe('AIAssistantConversationsDataClient', () => { { content: 'Go ahead and click the add connector button below to continue the conversation!', - presentation: { - delay: 1000, - stream: true, - }, role: 'assistant', timestamp: '1/24/2024, 5:32:19 PM', }, @@ -160,10 +149,12 @@ describe('AIAssistantConversationsDataClient', () => { timestamp: '2024-01-25T01:32:37.649Z', title: 'Alert summary', updatedAt: '2024-01-25T01:34:51.303Z', - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + users: [ + { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, + ], }); }); @@ -203,9 +194,7 @@ describe('AIAssistantConversationsDataClient', () => { '@timestamp': '2019-12-13T16:40:33.400Z', content: 'test content', is_error: undefined, - presentation: undefined, reader: undefined, - replacements: undefined, role: 'user', trace_data: { trace_id: '1', diff --git a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts index baee2da29adcf..ed6edae590c99 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.test.ts @@ -93,7 +93,7 @@ describe('DocumentsDataWriter', () => { expect(errors).toEqual([ { - conversation: { + document: { id: '', }, message: 'something went wrong', @@ -174,7 +174,7 @@ describe('DocumentsDataWriter', () => { expect(errors).toEqual([ { - conversation: { + document: { id: undefined, }, message: 'something went wrong', diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index cb729efb4b902..82e21a8cd8690 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -87,7 +87,6 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, - telemetry: core.analytics, }); }), @@ -97,7 +96,6 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, - telemetry: core.analytics, }); }), @@ -107,7 +105,6 @@ export class RequestContextFactory implements IRequestContextFactory { spaceId: getSpaceId(), logger: this.logger, currentUser, - telemetry: core.analytics, }); }), }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts index 8a678aa017690..e98cac07cd68c 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/create_route.ts @@ -53,7 +53,7 @@ export const createConversationRoute = (router: ElasticAssistantPluginRouter): v const result = await dataClient?.findConversations({ perPage: 100, page: 1, - filter: `user.id:${authenticatedUser?.profile_uid} AND title:${request.body.title}`, + filter: `users:{ id: "${authenticatedUser?.profile_uid}" } AND title:${request.body.title}`, fields: ['title'], }); if (result?.data != null && result.data.length > 0) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts index 5c845e90b38fb..0a9343b8944da 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/user_conversations/find_user_conversations_route.ts @@ -46,13 +46,13 @@ export const findUserConversationsRoute = (router: ElasticAssistantPluginRouter) const dataClient = await ctx.elasticAssistant.getAIAssistantConversationsDataClient(); const currentUser = ctx.elasticAssistant.getCurrentUser(); - const additionalFilter = query.filter ? `AND ${query.filter}` : ''; + const additionalFilter = query.filter ? ` AND ${query.filter}` : ''; const result = await dataClient?.findConversations({ perPage: query.per_page, page: query.page, sortField: query.sort_field, sortOrder: query.sort_order, - filter: `users:{ id: "${currentUser?.profile_uid}${additionalFilter}" }`, + filter: `users:{ id: "${currentUser?.profile_uid}" }${additionalFilter}`, fields: query.fields, }); From 5d281a19703e8b249f137430a844dd0a30fd370c Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 18:51:46 -0800 Subject: [PATCH 085/141] fixed typechecks --- .../elastic_assistant/server/__mocks__/request.ts | 7 ------- .../server/__mocks__/request_context.ts | 8 ++++---- .../delete_conversation.test.ts | 10 ++++++---- .../server/lib/data_client/documents_data_writer.ts | 1 - .../group2/tests/actions/connector_types/bedrock.ts | 4 ++++ 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index ad0370207e404..6b42d9d63043f 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -14,7 +14,6 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, - ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, PostEvaluateRequestBodyInput, PostEvaluateRequestQueryInput, } from '@kbn/elastic-assistant-common'; @@ -70,12 +69,6 @@ export const getPostEvaluateRequest = ({ }); export const getCurrentUserFindRequest = () => - requestMock.create({ - method: 'get', - path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND_USER_CONVERSATIONS, - }); - -export const getFindRequest = () => requestMock.create({ method: 'get', path: ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index 9b34184d0c1f6..64dcd75a1f859 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -31,8 +31,8 @@ export const createMockClients = () => { logger: loggingSystemMock.createLogger(), telemetry: coreMock.createSetup().analytics, getAIAssistantConversationsDataClient: conversationsDataClientMock.create(), - getAIAssistantPromptsSOClient: jest.fn(), - getAIAssistantAnonymizationFieldsSOClient: jest.fn(), + getAIAssistantPromptsDataClient: jest.fn(), + getAIAssistantAnonymizationFieldsDataClient: jest.fn(), getSpaceId: jest.fn(), getCurrentUser: jest.fn(), }, @@ -95,8 +95,8 @@ const createElasticAssistantRequestContextMock = ( > & (() => Promise), - getAIAssistantPromptsSOClient: jest.fn(), - getAIAssistantAnonymizationFieldsSOClient: jest.fn(), + getAIAssistantAnonymizationFieldsDataClient: jest.fn(), + getAIAssistantPromptsDataClient: jest.fn(), getCurrentUser: jest.fn(), getServerBasePath: jest.fn(), getSpaceId: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index 371f8a8d4c3c4..ee056c211aecb 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -34,10 +34,12 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: Date.now().toLocaleString(), timestamp: Date.now().toLocaleString(), - user: { - id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', - name: 'elastic', - }, + users: [ + { + id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', + name: 'elastic', + }, + ], }); export const getDeleteConversationOptionsMock = (): DeleteConversationParams => ({ diff --git a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts index 7211550d97bba..76671f31e0aa4 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/data_client/documents_data_writer.ts @@ -246,7 +246,6 @@ export class DocumentsDataWriter implements DocumentsDataWriter { private buildBulkOperations = async ( params: BulkParams ): Promise => { - const changedAt = new Date().toISOString(); const documentCreateBody = params.authenticatedUser && params.documentsToCreate ? params.documentsToCreate.flatMap((document) => [ diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index f231ea5612c3e..d0ea185be5f8b 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -444,6 +444,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { }, isEnabledKnowledgeBase: false, isEnabledRAGAlerts: false, + llmType: 'bedrock', }) .pipe(passThrough); const responseBuffer: Uint8Array[] = []; @@ -469,6 +470,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + llmType: 'bedrock', subAction: 'getDashboard', subActionParams: { dashboardId, @@ -495,6 +497,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + llmType: 'bedrock', subAction: 'getDashboard', subActionParams: { dashboardId, @@ -572,6 +575,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + llmType: 'bedrock', subAction: 'test', subActionParams: { body: JSON.stringify(DEFAULT_BODY), From 29a78e6a498607a1fee90e6bea27b46f2ce9171b Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 20:34:19 -0800 Subject: [PATCH 086/141] test fix --- .../group2/tests/actions/connector_types/bedrock.ts | 10 +++++++--- .../group2/tests/actions/connector_types/openai.ts | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index d0ea185be5f8b..4c570cf756c71 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -282,6 +282,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { subAction: 'invalidAction' }, }) .expect(200); @@ -326,6 +327,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { subAction: 'test', subActionParams: { @@ -356,6 +358,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { subAction: 'test', subActionParams: { @@ -380,6 +383,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { subAction: 'invokeAI', subActionParams: { @@ -469,8 +473,8 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .auth('global_read', 'global_read-password') .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { - llmType: 'bedrock', subAction: 'getDashboard', subActionParams: { dashboardId, @@ -496,8 +500,8 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { - llmType: 'bedrock', subAction: 'getDashboard', subActionParams: { dashboardId, @@ -574,8 +578,8 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'bedrock', params: { - llmType: 'bedrock', subAction: 'test', subActionParams: { body: JSON.stringify(DEFAULT_BODY), diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts index 4d23e887085e0..d279488ae1256 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts @@ -258,6 +258,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'openai', params: { subAction: 'invalidAction' }, }) .expect(200); @@ -297,6 +298,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'openai', params: { subAction: 'test', subActionParams: { @@ -325,6 +327,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { .auth('global_read', 'global_read-password') .set('kbn-xsrf', 'foo') .send({ + llmType: 'openai', params: { subAction: 'getDashboard', subActionParams: { @@ -403,6 +406,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`${getUrlPrefix('space1')}/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'openai', params: { subAction: 'getDashboard', subActionParams: { @@ -473,6 +477,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ + llmType: 'openai', params: { subAction: 'test', subActionParams: { From b4987b09df42c2d5c26365101dc401d411284ed8 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 21:35:18 -0800 Subject: [PATCH 087/141] added category and summary fields --- .../conversations/common_attributes.gen.ts | 50 +++++++++++++++++++ .../common_attributes.schema.yaml | 47 ++++++++++++++++- .../conversation_selector/index.tsx | 1 + .../conversation_settings.tsx | 1 + .../impl/assistant/helpers.ts | 1 - .../impl/assistant/index.test.tsx | 10 ++++ .../system_prompt/index.test.tsx | 1 + .../use_conversation/helpers.test.ts | 6 +++ .../impl/assistant/use_conversation/index.tsx | 2 + .../use_conversation/sample_conversations.tsx | 1 + .../impl/assistant_context/types.tsx | 1 + .../connector_selector_inline.test.tsx | 4 ++ .../impl/mock/conversation.ts | 3 ++ .../__mocks__/conversations_schema.mock.ts | 4 ++ .../append_conversation_messages.ts | 2 - .../conversations_configuration_type.ts | 36 ++++++++++--- .../conversations_data_writer.ts | 2 - .../create_conversation.test.ts | 3 ++ .../create_conversation.ts | 5 ++ .../delete_conversation.test.ts | 1 + .../get_conversation.test.ts | 8 +++ .../conversations_data_client/transforms.ts | 2 + .../server/conversations_data_client/types.ts | 16 +++++- .../update_conversation.ts | 4 +- .../assistant/content/conversations/index.tsx | 6 +++ 25 files changed, 200 insertions(+), 17 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts index 4121e90465edf..dadeb51c4d008 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.gen.ts @@ -83,6 +83,22 @@ export const MessageRole = z.enum(['system', 'user', 'assistant']); export type MessageRoleEnum = typeof MessageRole.enum; export const MessageRoleEnum = MessageRole.enum; +/** + * The conversation category. + */ +export type ConversationCategory = z.infer; +export const ConversationCategory = z.enum(['assistant', 'insights']); +export type ConversationCategoryEnum = typeof ConversationCategory.enum; +export const ConversationCategoryEnum = ConversationCategory.enum; + +/** + * The conversation confidence. + */ +export type ConversationConfidence = z.infer; +export const ConversationConfidence = z.enum(['low', 'medium', 'high']); +export type ConversationConfidenceEnum = typeof ConversationConfidence.enum; +export const ConversationConfidenceEnum = ConversationConfidence.enum; + /** * AI assistant conversation message. */ @@ -138,6 +154,26 @@ export const ApiConfig = z.object({ model: z.string().optional(), }); +export type ConversationSummary = z.infer; +export const ConversationSummary = z.object({ + /** + * Summary text of the conversation over time. + */ + content: z.string().optional(), + /** + * The timestamp summary was updated. + */ + timestamp: NonEmptyString.optional(), + /** + * Define if summary is marked as publicly available. + */ + public: z.boolean().optional(), + /** + * How confident you are about this being a correct and useful learning. + */ + confidence: ConversationConfidence.optional(), +}); + export type ErrorSchema = z.infer; export const ErrorSchema = z .object({ @@ -156,6 +192,11 @@ export const ConversationResponse = z.object({ * The conversation title. */ title: z.string(), + /** + * The conversation category. + */ + category: ConversationCategory, + summary: ConversationSummary.optional(), timestamp: NonEmptyString.optional(), /** * The last time conversation was updated. @@ -196,6 +237,10 @@ export const ConversationUpdateProps = z.object({ * The conversation title. */ title: z.string().optional(), + /** + * The conversation category. + */ + category: ConversationCategory.optional(), /** * The conversation messages. */ @@ -204,6 +249,7 @@ export const ConversationUpdateProps = z.object({ * LLM API configuration. */ apiConfig: ApiConfig.optional(), + summary: ConversationSummary.optional(), /** * excludeFromLastConversationStorage. */ @@ -217,6 +263,10 @@ export const ConversationCreateProps = z.object({ * The conversation title. */ title: z.string(), + /** + * The conversation category. + */ + category: ConversationCategory.optional(), /** * The conversation messages. */ diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml index 3fd470fc9e78e..e25a243829667 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/conversations/common_attributes.schema.yaml @@ -62,6 +62,21 @@ components: - user - assistant + ConversationCategory: + type: string + description: The conversation category. + enum: + - assistant + - insights + + ConversationConfidence: + type: string + description: The conversation confidence. + enum: + - low + - medium + - high + Message: type: object description: AI assistant conversation message. @@ -106,7 +121,23 @@ components: description: Provider model: type: string - description: model + description: model + + ConversationSummary: + type: object + properties: + content: + type: string + description: Summary text of the conversation over time. + timestamp: + $ref: '#/components/schemas/NonEmptyString' + description: The timestamp summary was updated. + public: + type: boolean + description: Define if summary is marked as publicly available. + confidence: + $ref: '#/components/schemas/ConversationConfidence' + description: How confident you are about this being a correct and useful learning. ErrorSchema: type: object @@ -137,6 +168,7 @@ components: - users - namespace - apiConfig + - category properties: id: oneOf: @@ -145,6 +177,11 @@ components: title: type: string description: The conversation title. + category: + $ref: '#/components/schemas/ConversationCategory' + description: The conversation category. + summary: + $ref: '#/components/schemas/ConversationSummary' 'timestamp': $ref: '#/components/schemas/NonEmptyString' updatedAt: @@ -189,6 +226,9 @@ components: title: type: string description: The conversation title. + category: + $ref: '#/components/schemas/ConversationCategory' + description: The conversation category. messages: type: array items: @@ -197,6 +237,8 @@ components: apiConfig: $ref: '#/components/schemas/ApiConfig' description: LLM API configuration. + summary: + $ref: '#/components/schemas/ConversationSummary' excludeFromLastConversationStorage: description: excludeFromLastConversationStorage. type: boolean @@ -211,6 +253,9 @@ components: title: type: string description: The conversation title. + category: + $ref: '#/components/schemas/ConversationCategory' + description: The conversation category. messages: type: array items: diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx index 3b12c7245fd91..b8c1f8e5476e7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_selector/index.tsx @@ -111,6 +111,7 @@ export const ConversationSelector: React.FC = React.memo( const newConversation: Conversation = { id: searchValue, title: searchValue, + category: 'assistant', messages: [], apiConfig: { connectorId: defaultConnectorId, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx index e3713cbb181f1..5717dbabb8558 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.tsx @@ -77,6 +77,7 @@ export const ConversationSettings: React.FC = React.m ? { id: c ?? '', title: c ?? '', + category: 'assistant', messages: [], apiConfig: { connectorId: defaultConnectorId, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index bff5fbec68252..cd76ebfc20dd0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -107,7 +107,6 @@ interface OptionalRequestParams { alertsIndexPattern?: string; allow?: string[]; allowReplacement?: string[]; - replacements?: Record; size?: number; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx index d00f6926a58f5..e6a03b0819963 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.test.tsx @@ -78,12 +78,14 @@ describe('Assistant', () => { Welcome: { id: 'Welcome Id', title: 'Welcome', + category: 'assistant', messages: [], apiConfig: {}, }, 'electric sheep': { id: 'electric sheep id', title: 'electric sheep', + category: 'assistant', messages: [], apiConfig: {}, }, @@ -121,11 +123,13 @@ describe('Assistant', () => { Welcome: { id: 'Welcome Id', title: 'Welcome', + category: 'assistant', messages: [], apiConfig: {}, }, 'electric sheep': { id: 'electric sheep id', + category: 'assistant', title: 'electric sheep', messages: [], apiConfig: {}, @@ -164,11 +168,13 @@ describe('Assistant', () => { Welcome: { id: 'Welcome Id', title: 'Welcome', + category: 'assistant', messages: [], apiConfig: {}, }, 'electric sheep': { id: 'electric sheep id', + category: 'assistant', title: 'electric sheep', messages: [], apiConfig: {}, @@ -202,12 +208,14 @@ describe('Assistant', () => { Welcome: { id: 'Welcome', title: 'Welcome', + category: 'assistant', messages: [], apiConfig: {}, }, 'electric sheep': { id: 'electric sheep', title: 'electric sheep', + category: 'assistant', messages: [], apiConfig: {}, }, @@ -230,10 +238,12 @@ describe('Assistant', () => { Welcome: { id: 'Welcome', title: 'Welcome', + category: 'assistant', messages: [], apiConfig: {}, }, 'electric sheep': { + category: 'assistant', id: 'electric sheep', title: 'electric sheep', messages: [], diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx index 3323f05b062fe..576ec739045e6 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.test.tsx @@ -372,6 +372,7 @@ describe('SystemPrompt', () => { it('should save new prompt correctly when prompt is removed from a conversation and linked to another conversation in a single transaction', async () => { const secondMockConversation: Conversation = { id: 'second', + category: 'assistant', apiConfig: { defaultSystemPromptId: undefined, }, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts index 28f03dd4aab6e..3ecf24c2f5263 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts @@ -94,6 +94,7 @@ describe('useConversation helpers', () => { apiConfig: { defaultSystemPromptId: '3', }, + category: 'assistant', id: '1', messages: [], title: '1', @@ -108,6 +109,7 @@ describe('useConversation helpers', () => { test('should return the default (starred) isNewConversationDefault system prompt if conversation system prompt does not exist', () => { const conversationWithoutSystemPrompt: Conversation = { apiConfig: {}, + category: 'assistant', id: '1', messages: [], title: '1', @@ -123,6 +125,7 @@ describe('useConversation helpers', () => { test('should return the default (starred) isNewConversationDefault system prompt if conversation system prompt does not exist within all system prompts', () => { const conversationWithoutSystemPrompt: Conversation = { apiConfig: {}, + category: 'assistant', id: '4', // this id does not exist within allSystemPrompts messages: [], title: '4', @@ -138,6 +141,7 @@ describe('useConversation helpers', () => { test('should return the first prompt if both conversation system prompt and default new system prompt do not exist', () => { const conversationWithoutSystemPrompt: Conversation = { apiConfig: {}, + category: 'assistant', id: '1', messages: [], title: '1', @@ -153,6 +157,7 @@ describe('useConversation helpers', () => { test('should return undefined if conversation system prompt does not exist and there are no system prompts', () => { const conversationWithoutSystemPrompt: Conversation = { apiConfig: {}, + category: 'assistant', id: '1', messages: [], title: '1', @@ -168,6 +173,7 @@ describe('useConversation helpers', () => { test('should return undefined if conversation system prompt does not exist within all system prompts', () => { const conversationWithoutSystemPrompt: Conversation = { apiConfig: {}, + category: 'assistant', id: '4', // this id does not exist within allSystemPrompts messages: [], title: '1', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index ed26f661e3a4d..f0a3a913cb1c5 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -23,6 +23,7 @@ export const DEFAULT_CONVERSATION_STATE: Conversation = { id: i18n.DEFAULT_CONVERSATION_TITLE, messages: [], apiConfig: {}, + category: 'assistant', title: i18n.DEFAULT_CONVERSATION_TITLE, }; @@ -162,6 +163,7 @@ export const useConversation = (): UseConversation => { http, conversation: { apiConfig, + category: 'assistant', title: conversation.title, replacements: conversation.replacements, excludeFromLastConversationStorage: conversation.excludeFromLastConversationStorage, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx index a3898e1edb9e9..5b2d54f32f774 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/sample_conversations.tsx @@ -12,6 +12,7 @@ import { WELCOME_CONVERSATION_TITLE } from './translations'; export const WELCOME_CONVERSATION: Conversation = { id: WELCOME_CONVERSATION_TITLE, title: WELCOME_CONVERSATION_TITLE, + category: 'assistant', messages: [ { role: 'assistant', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 0fd7f187b02a4..117ef288472af 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -63,6 +63,7 @@ export interface Conversation { id?: string; name?: string; }; + category: string; id: string; title: string; messages: Message[]; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx index 19b5f24e221c3..52a2c33e58f1a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx @@ -75,6 +75,7 @@ describe('ConnectorSelectorInline', () => { it('renders empty view if selectedConnectorId is NOT in list of connectors', () => { const conversation: Conversation = { id: 'conversation_id', + category: 'assistant', messages: [], apiConfig: {}, title: 'conversation_id', @@ -94,6 +95,7 @@ describe('ConnectorSelectorInline', () => { it('Clicking add connector button opens the connector selector', () => { const conversation: Conversation = { id: 'conversation_id', + category: 'assistant', messages: [], apiConfig: {}, title: 'conversation_id', @@ -116,6 +118,7 @@ describe('ConnectorSelectorInline', () => { const connectorTwo = mockConnectors[1]; const conversation: Conversation = { id: 'conversation_id', + category: 'assistant', messages: [], apiConfig: {}, title: 'conversation_id', @@ -152,6 +155,7 @@ describe('ConnectorSelectorInline', () => { it('On connector change to add new connector, onchange event does nothing', () => { const conversation: Conversation = { id: 'conversation_id', + category: 'assistant', messages: [], apiConfig: {}, title: 'conversation_id', diff --git a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts index 23ecacc7d9cf7..de2340879428e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/mock/conversation.ts @@ -11,6 +11,7 @@ import { Conversation } from '../..'; export const alertConvo: Conversation = { id: 'Alert summary', title: 'Alert summary', + category: 'assistant', isDefault: true, messages: [ { @@ -34,6 +35,7 @@ export const alertConvo: Conversation = { export const emptyWelcomeConvo: Conversation = { id: 'Welcome', title: 'Welcome', + category: 'assistant', isDefault: true, messages: [], apiConfig: { @@ -62,6 +64,7 @@ export const welcomeConvo: Conversation = { export const customConvo: Conversation = { id: 'Custom option', + category: 'assistant', title: 'Custom option', isDefault: false, messages: [], diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts index 647f466a3d11a..2d1560b245019 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/conversations_schema.mock.ts @@ -34,6 +34,7 @@ export const getCreateConversationSchemaMock = (): ConversationCreateProps => ({ }, }, ], + category: 'assistant', }); export const getUpdateConversationSchemaMock = ( @@ -86,6 +87,7 @@ export const getConversationMock = ( createdAt: '2019-12-13T16:40:33.400Z', updatedAt: '2019-12-13T16:40:33.400Z', namespace: 'default', + category: 'assistant', users: [ { name: 'elastic', @@ -105,6 +107,7 @@ export const getQueryConversationParams = ( connectorTypeTitle: 'Test connector', model: 'model', }, + category: 'assistant', excludeFromLastConversationStorage: false, messages: [ { @@ -121,6 +124,7 @@ export const getQueryConversationParams = ( } : { title: 'Welcome', + category: 'assistant', apiConfig: { connectorId: '1', defaultSystemPromptId: 'Default', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts index 6401f4133d635..67e86108f88a8 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/append_conversation_messages.ts @@ -54,9 +54,7 @@ export const appendConversationMessages = async ({ newMessage['@timestamp'] = message['@timestamp']; newMessage.content = message.content; newMessage.is_error = message.is_error; - newMessage.presentation = message.presentation; newMessage.reader = message.reader; - newMessage.replacements = message.replacements; newMessage.role = message.role; messages.add(newMessage); } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts index 3a941c51fbe5f..926fadae1eff4 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_configuration_type.ts @@ -53,12 +53,12 @@ export const conversationsFieldMap: FieldMap = { required: false, }, messages: { - type: 'object', + type: 'nested', array: true, required: false, }, 'messages.@timestamp': { - type: 'keyword', + type: 'date', array: false, required: true, }, @@ -73,7 +73,7 @@ export const conversationsFieldMap: FieldMap = { required: false, }, 'messages.content': { - type: 'keyword', + type: 'text', array: false, required: false, }, @@ -82,11 +82,6 @@ export const conversationsFieldMap: FieldMap = { array: false, required: false, }, - 'messages.replacements': { - type: 'object', - array: false, - required: false, - }, 'messages.trace_data': { type: 'object', array: false, @@ -102,6 +97,31 @@ export const conversationsFieldMap: FieldMap = { array: false, required: false, }, + summary: { + type: 'object', + array: false, + required: false, + }, + 'summary.content': { + type: 'text', + array: false, + required: false, + }, + 'summary.@timestamp': { + type: 'date', + array: false, + required: true, + }, + 'summary.public': { + type: 'boolean', + array: false, + required: false, + }, + 'summary.confidence': { + type: 'keyword', + array: false, + required: false, + }, api_config: { type: 'object', array: false, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts index de790c6633820..dada6c0298888 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/conversations_data_writer.ts @@ -210,9 +210,7 @@ export class ConversationDataWriter implements ConversationDataWriter { newMessage['@timestamp'] = message['@timestamp']; newMessage.content = message.content; newMessage.is_error = message.is_error; - newMessage.presentation = message.presentation; newMessage.reader = message.reader; - newMessage.replacements = message.replacements; newMessage.role = message.role; messages.add(newMessage); } diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts index 6161eea8deff3..ce7820400f028 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.test.ts @@ -40,6 +40,7 @@ export const getCreateConversationMock = (): ConversationCreateProps => ({ messages: [], // eslint-disable-next-line @typescript-eslint/no-explicit-any replacements: {} as any, + category: 'assistant', }); export const getConversationResponseMock = (): ConversationResponse => ({ @@ -61,6 +62,7 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: '2024-01-28T04:20:02.394Z', timestamp: '2024-01-28T04:20:02.394Z', + category: 'assistant', users: [ { name: 'test', @@ -89,6 +91,7 @@ export const getSearchConversationMock = title: 'title-1', updated_at: '2020-04-20T15:25:31.830Z', messages: [], + category: 'assistant', id: '1', namespace: 'default', is_default: true, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts index f0013017c07ab..2cfd09d9a198f 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/create_conversation.ts @@ -9,6 +9,8 @@ import { v4 as uuidv4 } from 'uuid'; import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { + ConversationCategory, + ConversationCategoryEnum, ConversationCreateProps, ConversationResponse, MessageRole, @@ -25,6 +27,7 @@ export interface CreateMessageSchema { created_at: string; title: string; id?: string | undefined; + category: ConversationCategory; messages?: Array<{ '@timestamp': string; content: string; @@ -102,6 +105,7 @@ export const transformToCreateScheme = ( { title, apiConfig, + category, excludeFromLastConversationStorage, isDefault, messages, @@ -118,6 +122,7 @@ export const transformToCreateScheme = ( }, ], title, + category: category ?? ConversationCategoryEnum.assistant, api_config: { connector_id: apiConfig?.connectorId, connector_type_title: apiConfig?.connectorTypeTitle, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts index ee056c211aecb..da93f40233f7b 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/delete_conversation.test.ts @@ -34,6 +34,7 @@ export const getConversationResponseMock = (): ConversationResponse => ({ isDefault: false, updatedAt: Date.now().toLocaleString(), timestamp: Date.now().toLocaleString(), + category: 'assistant', users: [ { id: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts index 68a0db9250600..1ea3962c9def9 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/get_conversation.test.ts @@ -31,6 +31,10 @@ export const getConversationResponseMock = (): ConversationResponse => ({ model: 'test', provider: 'Azure OpenAI', }, + summary: { + content: 'test', + }, + category: 'assistant', users: [ { id: '1111', @@ -80,6 +84,10 @@ export const getSearchConversationMock = model: 'test', provider: 'Azure OpenAI', }, + summary: { + content: 'test', + }, + category: 'assistant', users: [ { id: '1111', diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts index 398c7dd96c126..9790d1ee827b2 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/transforms.ts @@ -26,6 +26,8 @@ export const transformESToConversations = ( name: user.name, })) ?? [], title: conversationSchema.title, + category: conversationSchema.category, + summary: conversationSchema.summary, apiConfig: { connectorId: conversationSchema.api_config?.connector_id, connectorTypeTitle: conversationSchema.api_config?.connector_type_title, diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts index 1a65f56dad0ee..bb73b2adb502a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/types.ts @@ -5,13 +5,27 @@ * 2.0. */ -import { MessageRole, Provider, Reader, Replacement } from '@kbn/elastic-assistant-common'; +import { + ConversationCategory, + ConversationConfidence, + MessageRole, + Provider, + Reader, + Replacement, +} from '@kbn/elastic-assistant-common'; export interface SearchEsConversationSchema { id: string; '@timestamp': string; created_at: string; title: string; + summary?: { + content?: string; + timestamp?: string; + public?: boolean; + confidence?: ConversationConfidence; + }; + category: ConversationCategory; messages?: Array<{ '@timestamp': string; content: string; diff --git a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts index debeddd9dc43a..a21c11f1c2d1a 100644 --- a/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts +++ b/x-pack/plugins/elastic_assistant/server/conversations_data_client/update_conversation.ts @@ -14,6 +14,7 @@ import { Provider, MessageRole, getMessageContentWithoutReplacements, + ConversationSummary, } from '@kbn/elastic-assistant-common'; import { AuthenticatedUser } from '@kbn/security-plugin/common'; import { getConversation } from './get_conversation'; @@ -39,6 +40,7 @@ export interface UpdateConversationSchema { provider?: Provider; model?: string; }; + summary?: ConversationSummary; exclude_from_last_conversation_storage?: boolean; replacements?: Replacement; updated_at?: string; @@ -118,9 +120,7 @@ export const updateConversation = async ({ newMessage['@timestamp'] = message['@timestamp']; newMessage.content = message.content; newMessage.is_error = message.is_error; - newMessage.presentation = message.presentation; newMessage.reader = message.reader; - newMessage.replacements = message.replacements; newMessage.role = message.role; messages.add(newMessage); } diff --git a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx index 2216eb04fe294..12b3d2490bfd8 100644 --- a/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/content/conversations/index.tsx @@ -19,6 +19,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [ALERT_SUMMARY_CONVERSATION_ID]: { id: ALERT_SUMMARY_CONVERSATION_ID, title: ALERT_SUMMARY_CONVERSATION_ID, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, @@ -26,6 +27,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [DATA_QUALITY_DASHBOARD_CONVERSATION_ID]: { id: DATA_QUALITY_DASHBOARD_CONVERSATION_ID, title: DATA_QUALITY_DASHBOARD_CONVERSATION_ID, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, @@ -33,6 +35,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [DETECTION_RULES_CONVERSATION_ID]: { id: DETECTION_RULES_CONVERSATION_ID, title: DETECTION_RULES_CONVERSATION_ID, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, @@ -40,6 +43,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [EVENT_SUMMARY_CONVERSATION_ID]: { id: EVENT_SUMMARY_CONVERSATION_ID, title: EVENT_SUMMARY_CONVERSATION_ID, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, @@ -48,6 +52,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { excludeFromLastConversationStorage: true, id: TIMELINE_CONVERSATION_TITLE, title: TIMELINE_CONVERSATION_TITLE, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, @@ -55,6 +60,7 @@ export const BASE_SECURITY_CONVERSATIONS: Record = { [WELCOME_CONVERSATION_TITLE]: { id: WELCOME_CONVERSATION_TITLE, title: WELCOME_CONVERSATION_TITLE, + category: 'assistant', isDefault: true, messages: [], apiConfig: {}, From 3f607ff73df12035be52f0436edbe59d5b1744a8 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Tue, 13 Feb 2024 21:43:12 -0800 Subject: [PATCH 088/141] test fix --- .../group2/tests/actions/connector_types/openai.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts index d279488ae1256..4d23e887085e0 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts @@ -258,7 +258,6 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'openai', params: { subAction: 'invalidAction' }, }) .expect(200); @@ -298,7 +297,6 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'openai', params: { subAction: 'test', subActionParams: { @@ -327,7 +325,6 @@ export default function genAiTest({ getService }: FtrProviderContext) { .auth('global_read', 'global_read-password') .set('kbn-xsrf', 'foo') .send({ - llmType: 'openai', params: { subAction: 'getDashboard', subActionParams: { @@ -406,7 +403,6 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`${getUrlPrefix('space1')}/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'openai', params: { subAction: 'getDashboard', subActionParams: { @@ -477,7 +473,6 @@ export default function genAiTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${genAiActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'openai', params: { subAction: 'test', subActionParams: { From 624847e5d2fc207318c3d3709bf71dd693eb6b22 Mon Sep 17 00:00:00 2001 From: YulNaumenko Date: Wed, 14 Feb 2024 08:47:46 -0800 Subject: [PATCH 089/141] tests --- .../conversation_settings/conversation_settings.test.tsx | 1 + .../kbn-elastic-assistant/impl/assistant/helpers.test.ts | 6 ++++++ .../impl/assistant/use_conversation/index.test.tsx | 1 + .../connector_selector_inline.test.tsx | 1 + .../group2/tests/actions/connector_types/bedrock.ts | 7 ------- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx index f70c4acecfcbf..25b1a9c95cea1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx @@ -151,6 +151,7 @@ describe('ConversationSettings', () => { defaultSystemPromptId: 'default-system-prompt', provider: 'OpenAI', }, + category: 'assistant', id: 'Cool new conversation', title: 'Cool new conversation', messages: [], diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts index bed90bdf6c215..9edd361300968 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts @@ -21,6 +21,7 @@ describe('getBlockBotConversation', () => { it('When no conversation history, return only enterprise messaging', () => { const conversation = { id: 'conversation_id', + category: 'assistant', theme: {}, messages: [], apiConfig: {}, @@ -46,6 +47,7 @@ describe('getBlockBotConversation', () => { }, ], apiConfig: {}, + category: 'assistant', title: 'conversation_id', }; const result = getBlockBotConversation(conversation, isAssistantEnabled); @@ -58,6 +60,7 @@ describe('getBlockBotConversation', () => { title: 'conversation_id', messages: enterpriseMessaging, apiConfig: {}, + category: 'assistant', }; const result = getBlockBotConversation(conversation, isAssistantEnabled); expect(result.messages.length).toEqual(1); @@ -68,6 +71,7 @@ describe('getBlockBotConversation', () => { const conversation = { id: 'conversation_id', title: 'conversation_id', + category: 'assistant', messages: [ ...enterpriseMessaging, { @@ -93,6 +97,7 @@ describe('getBlockBotConversation', () => { const conversation = { id: 'conversation_id', title: 'conversation_id', + category: 'assistant', messages: [], apiConfig: {}, }; @@ -103,6 +108,7 @@ describe('getBlockBotConversation', () => { const conversation = { id: 'conversation_id', title: 'conversation_id', + category: 'assistant', messages: [ { role: 'user' as const, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index d117579443146..518bccf86b576 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -64,6 +64,7 @@ describe('useConversation', () => { messages: mockConvo.messages, apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, title: mockConvo.title, + category: 'assistant', }); expect(createResult).toEqual(mockConvo); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx index 52a2c33e58f1a..501fdea88a790 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/connector_selector_inline.test.tsx @@ -146,6 +146,7 @@ describe('ConnectorSelectorInline', () => { }, conversation: { apiConfig: {}, + category: 'assistant', id: 'conversation_id', messages: [], title: 'conversation_id', diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index 4c570cf756c71..e368a605973ca 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -282,7 +282,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'invalidAction' }, }) .expect(200); @@ -327,7 +326,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'test', subActionParams: { @@ -358,7 +356,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'test', subActionParams: { @@ -383,7 +380,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'invokeAI', subActionParams: { @@ -473,7 +469,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .auth('global_read', 'global_read-password') .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'getDashboard', subActionParams: { @@ -500,7 +495,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'getDashboard', subActionParams: { @@ -578,7 +572,6 @@ export default function bedrockTest({ getService }: FtrProviderContext) { .post(`/api/actions/connector/${bedrockActionId}/_execute`) .set('kbn-xsrf', 'foo') .send({ - llmType: 'bedrock', params: { subAction: 'test', subActionParams: { From f8a83e8d9b0b7c664a3b1b3027c308dc930d26b3 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 14 Feb 2024 12:07:19 -0700 Subject: [PATCH 090/141] update replacements --- .../server/routes/post_actions_connector_execute.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index 0f2aa5926cb61..21d1334e4ac28 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -269,6 +269,13 @@ export const postActionsConnectorExecuteRoute = ( }), ], }); + await dataClient?.updateConversation({ + existingConversation: conversation, + conversationUpdateProps: { + id: conversation.id, + replacements: latestReplacements, + }, + }); } return response.ok({ body: { From cd1dd0932daec08ab6832bb095e39cbe50c59f5a Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Thu, 15 Feb 2024 09:16:15 -0700 Subject: [PATCH 091/141] more tests --- .../use_bulk_actions_conversations.test.ts | 163 ++++++++++++++++++ .../use_bulk_actions_conversations.ts | 3 +- .../assistant/assistant_header/index.test.tsx | 64 ++++++- .../impl/assistant/assistant_header/index.tsx | 20 ++- .../chat_send/use_chat_send.test.tsx | 32 ++++ .../conversation_settings.test.tsx | 97 ++++++++++- .../impl/assistant/helpers.test.ts | 160 ++++++++++++++++- 7 files changed, 522 insertions(+), 17 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.test.ts diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.test.ts new file mode 100644 index 0000000000000..0eb0d68c3ddb4 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { bulkChangeConversations } from './use_bulk_actions_conversations'; +import { + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, +} from '@kbn/elastic-assistant-common'; +import { httpServiceMock } from '@kbn/core-http-browser-mocks'; +import { IToasts } from '@kbn/core-notifications-browser'; + +const conversation1 = { + id: 'conversation1', + title: 'Conversation 1', + apiConfig: {}, + category: 'default', + messages: [ + { + id: 'message1', + role: 'user' as const, + content: 'Hello', + timestamp: '2024-02-14T19:58:30.299Z', + }, + { + id: 'message2', + role: 'user' as const, + content: 'How are you?', + timestamp: '2024-02-14T19:58:30.299Z', + }, + ], +}; +const conversation2 = { + ...conversation1, + id: 'conversation2', + title: 'Conversation 2', +}; +const toasts = { + addError: jest.fn(), +}; +describe('bulkChangeConversations', () => { + let httpMock: ReturnType; + + beforeEach(() => { + httpMock = httpServiceMock.createSetupContract(); + + jest.clearAllMocks(); + }); + it('should send a POST request with the correct parameters and receive a successful response', async () => { + const conversationsActions = { + create: {}, + update: {}, + delete: { ids: [] }, + }; + + await bulkChangeConversations(httpMock, conversationsActions); + + expect(httpMock.fetch).toHaveBeenCalledWith( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + { + method: 'POST', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + body: JSON.stringify({ + update: [], + create: [], + delete: { ids: [] }, + }), + } + ); + }); + + it('should transform the conversations dictionary to an array of conversations to create', async () => { + const conversationsActions = { + create: { + conversation1, + conversation2, + }, + update: {}, + delete: { ids: [] }, + }; + + await bulkChangeConversations(httpMock, conversationsActions); + + expect(httpMock.fetch).toHaveBeenCalledWith( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + { + method: 'POST', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + body: JSON.stringify({ + update: [], + create: [conversation1, conversation2], + delete: { ids: [] }, + }), + } + ); + }); + + it('should transform the conversations dictionary to an array of conversations to update', async () => { + const conversationsActions = { + update: { + conversation1, + conversation2, + }, + delete: { ids: [] }, + }; + + await bulkChangeConversations(httpMock, conversationsActions); + + expect(httpMock.fetch).toHaveBeenCalledWith( + ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BULK_ACTION, + { + method: 'POST', + version: ELASTIC_AI_ASSISTANT_API_CURRENT_VERSION, + body: JSON.stringify({ + update: [conversation1, conversation2], + delete: { ids: [] }, + }), + } + ); + }); + + it('should throw an error with the correct message when receiving an unsuccessful response', async () => { + httpMock.fetch.mockResolvedValue({ + success: false, + attributes: { + errors: [ + { + statusCode: 400, + message: 'Error updating conversations', + conversations: [{ id: conversation1.id, name: conversation1.title }], + }, + ], + }, + }); + const conversationsActions = { + create: {}, + update: {}, + delete: { ids: [] }, + }; + await bulkChangeConversations(httpMock, conversationsActions, toasts as unknown as IToasts); + expect(toasts.addError.mock.calls[0][0]).toEqual( + new Error('Error message: Error updating conversations for conversation Conversation 1') + ); + }); + + it('should handle cases where result.attributes.errors is undefined', async () => { + httpMock.fetch.mockResolvedValue({ + success: false, + attributes: {}, + }); + const conversationsActions = { + create: {}, + update: {}, + delete: { ids: [] }, + }; + + await bulkChangeConversations(httpMock, conversationsActions, toasts as unknown as IToasts); + expect(toasts.addError.mock.calls[0][0]).toEqual(new Error('')); + }); +}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts index 01cb406935d37..0886085c47b82 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/conversations/use_bulk_actions_conversations.ts @@ -128,11 +128,12 @@ export const bulkChangeConversations = async ( }), } ); + if (!result.success) { const serverError = result.attributes.errors ?.map( (e) => - `Error code: ${e.status_code}. Error message: ${ + `${e.status_code ? `Error code: ${e.status_code}. ` : ''}Error message: ${ e.message } for conversation ${e.conversations.map((c) => c.name).join(',')}` ) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx index d1ab1605f4f30..47b4f11629a9c 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.test.tsx @@ -6,13 +6,21 @@ */ import React from 'react'; -import { render } from '@testing-library/react'; +import { act, fireEvent, render } from '@testing-library/react'; import { AssistantHeader } from '.'; import { TestProviders } from '../../mock/test_providers/test_providers'; -import { alertConvo, emptyWelcomeConvo } from '../../mock/conversation'; +import { alertConvo, emptyWelcomeConvo, welcomeConvo } from '../../mock/conversation'; +import { useLoadConnectors } from '../../connectorland/use_load_connectors'; +import { mockConnectors } from '../../mock/connectors'; +const onConversationSelected = jest.fn(); +const setCurrentConversation = jest.fn(); +const mockConversations = { + [alertConvo.title]: alertConvo, + [welcomeConvo.title]: welcomeConvo, +}; const testProps = { - currentConversation: emptyWelcomeConvo, + currentConversation: welcomeConvo, title: 'Test Title', docLinks: { ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', @@ -20,18 +28,45 @@ const testProps = { }, isDisabled: false, isSettingsModalVisible: false, - onConversationSelected: jest.fn(), + onConversationSelected, onToggleShowAnonymizedValues: jest.fn(), selectedConversationId: emptyWelcomeConvo.id, setIsSettingsModalVisible: jest.fn(), - setCurrentConversation: jest.fn(), + setCurrentConversation, onConversationDeleted: jest.fn(), showAnonymizedValues: false, - conversations: {}, + conversations: mockConversations, refetchConversationsState: jest.fn(), }; +jest.mock('../../connectorland/use_load_connectors', () => ({ + useLoadConnectors: jest.fn(() => { + return { + data: [], + error: null, + isSuccess: true, + }; + }), +})); + +(useLoadConnectors as jest.Mock).mockReturnValue({ + data: mockConnectors, + error: null, + isSuccess: true, +}); +const mockSetApiConfig = alertConvo; +jest.mock('../use_conversation', () => ({ + useConversation: jest.fn(() => { + return { + setApiConfig: jest.fn().mockReturnValue(mockSetApiConfig), + }; + }), +})); + describe('AssistantHeader', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); it('showAnonymizedValues is not checked when currentConversation.replacements is null', () => { const { getByText, getByTestId } = render(, { wrapper: TestProviders, @@ -73,4 +108,21 @@ describe('AssistantHeader', () => { ); expect(getByTestId('showAnonymizedValues')).toHaveAttribute('aria-checked', 'true'); }); + + it('Conversation is updated when connector change occurs', async () => { + const { getByTestId } = render(, { + wrapper: TestProviders, + }); + fireEvent.click(getByTestId('connectorSelectorPlaceholderButton')); + fireEvent.click(getByTestId('connector-selector')); + + await act(async () => { + fireEvent.click(getByTestId('connectorId')); + }); + expect(setCurrentConversation).toHaveBeenCalledWith(alertConvo); + expect(onConversationSelected).toHaveBeenCalledWith({ + cId: alertConvo.id, + cTitle: alertConvo.title, + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index a1753662f333b..7ef8a06d47fb8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useMemo } from 'react'; +import React, { useCallback, useMemo } from 'react'; import { EuiFlexGroup, EuiFlexItem, @@ -74,6 +74,16 @@ export const AssistantHeader: React.FC = ({ showAnonymizedValues, [currentConversation.replacements, showAnonymizedValues] ); + const onConversationChange = useCallback( + (updatedConversation) => { + setCurrentConversation(updatedConversation); + onConversationSelected({ + cId: updatedConversation.id, + cTitle: updatedConversation.title, + }); + }, + [onConversationSelected, setCurrentConversation] + ); return ( <> = ({ isDisabled={isDisabled} docLinks={docLinks} selectedConversation={currentConversation} - onChange={(updatedConversation) => { - setCurrentConversation(updatedConversation); - onConversationSelected({ - cId: updatedConversation.id, - cTitle: updatedConversation.title, - }); - }} + onChange={onConversationChange} title={title} /> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index c91e2f2fa10a1..ca479f7f34282 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -14,9 +14,11 @@ import { useChatSend, UseChatSendProps } from './use_chat_send'; import { renderHook } from '@testing-library/react-hooks'; import { waitFor } from '@testing-library/react'; import { TestProviders } from '../../mock/test_providers/test_providers'; +import { useAssistantContext } from '../../..'; jest.mock('../use_send_messages'); jest.mock('../use_conversation'); +jest.mock('../../..'); const setEditingSystemPromptId = jest.fn(); const setPromptTextPreview = jest.fn(); @@ -49,6 +51,7 @@ export const testProps: UseChatSendProps = { setCurrentConversation, }; const robotMessage = { response: 'Response message from the robot', isError: false }; +const reportAssistantMessageSent = jest.fn(); describe('use chat send', () => { beforeEach(() => { jest.clearAllMocks(); @@ -60,6 +63,12 @@ describe('use chat send', () => { removeLastMessage, clearConversation, }); + (useAssistantContext as jest.Mock).mockReturnValue({ + assistantTelemetry: { + reportAssistantMessageSent, + }, + knowledgeBase: { isEnabledKnowledgeBase: false, isEnabledRAGAlerts: false }, + }); }); it('handleOnChatCleared clears the conversation', async () => { const { result } = renderHook(() => useChatSend(testProps), { @@ -136,4 +145,27 @@ describe('use chat send', () => { expect(messages[messages.length - 1].content).toEqual(robotMessage.response); }); }); + it('sends telemetry events for both user and assistant', async () => { + const promptText = 'prompt text'; + const { result } = renderHook(() => useChatSend(testProps), { + wrapper: TestProviders, + }); + result.current.handleButtonSendMessage(promptText); + expect(setUserPrompt).toHaveBeenCalledWith(''); + + await waitFor(() => { + expect(reportAssistantMessageSent).toHaveBeenNthCalledWith(1, { + conversationId: testProps.currentConversation.title, + role: 'user', + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: false, + }); + expect(reportAssistantMessageSent).toHaveBeenNthCalledWith(2, { + conversationId: testProps.currentConversation.title, + role: 'assistant', + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: false, + }); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx index 25b1a9c95cea1..7d2ddbbee99d3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_settings/conversation_settings.test.tsx @@ -22,6 +22,7 @@ const mockConvos = { const onSelectedConversationChange = jest.fn(); const setConversationSettings = jest.fn(); +const setConversationsSettingsBulkActions = jest.fn(); const testProps = { allSystemPrompts: mockSystemPrompts, @@ -33,7 +34,7 @@ const testProps = { selectedConversation: welcomeConvo, setConversationSettings, conversationsSettingsBulkActions: {}, - setConversationsSettingsBulkActions: jest.fn(), + setConversationsSettingsBulkActions, } as unknown as ConversationSettingsProps; jest.mock('../../../connectorland/use_load_connectors', () => ({ @@ -59,6 +60,11 @@ jest.mock('../conversation_selector_settings', () => ({ data-test-subj="change-convo" onClick={() => onConversationSelectionChange(mockConvo)} /> +