diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts index 2376b6c95b31a..7c14ee57805ec 100644 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts @@ -205,3 +205,12 @@ export enum ConversationAccess { SHARED = 'shared', PRIVATE = 'private', } + +export interface IntegrationKnowledgeBaseEntry { + content: string; + package_name: string; + filename: string; + version: string; + path: string; + installed_at: string; +} diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts index dba82fd3a69b4..1bae4b8a8c967 100644 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -13,7 +13,11 @@ import { encode } from 'gpt-tokenizer'; import { isLockAcquisitionError } from '@kbn/lock-manager'; import type { DocumentationManagerAPI } from '@kbn/product-doc-base-plugin/server/services/doc_manager'; import { resourceNames } from '..'; -import type { Instruction, KnowledgeBaseEntry } from '../../../common/types'; +import type { + Instruction, + IntegrationKnowledgeBaseEntry, + KnowledgeBaseEntry, +} from '../../../common/types'; import { KnowledgeBaseEntryRole, KnowledgeBaseType } from '../../../common/types'; import { getAccessQuery, getUserAccessFilters } from '../util/get_access_query'; import { getCategoryQuery } from '../util/get_category_query'; @@ -36,6 +40,8 @@ import { getInferenceIdFromWriteIndex } from './get_inference_id_from_write_inde import { createOrUpdateKnowledgeBaseIndexAssets } from '../index_assets/create_or_update_knowledge_base_index_assets'; import { LEGACY_CUSTOM_INFERENCE_ID } from '../../../common/preconfigured_inference_ids'; +const INTEGRATION_KNOWLEDGE_INDEX = '.integration_knowledge'; + interface Dependencies { core: CoreSetup; esClient: { @@ -112,6 +118,45 @@ export class KnowledgeBaseService { })); } + private async recallFromIntegrationsKnowledge({ + queries, + esClient, + }: { + queries: Array<{ text: string; boost?: number }>; + esClient: { asCurrentUser: ElasticsearchClient; asInternalUser: ElasticsearchClient }; + }): Promise { + try { + // Search the .integration_knowledge index using semantic search on the content field + const response = await esClient.asInternalUser.search({ + index: INTEGRATION_KNOWLEDGE_INDEX, + query: { + bool: { + should: queries.map(({ text, boost = 1 }) => ({ + semantic: { + field: 'content', + query: text, + boost, + }, + })), + }, + }, + size: 10, + _source: ['package_name', 'filename', 'content', 'version'], + }); + return response.hits.hits.map((hit) => ({ + text: hit._source?.content!, + labels: { filename: hit._source?.filename ?? '', version: hit._source?.version ?? '' }, + title: hit._source?.package_name, + esScore: hit._score!, + id: hit._id!, + })); + } catch (error) { + this.dependencies.logger.error(`Error recalling from ${INTEGRATION_KNOWLEDGE_INDEX} index`); + this.dependencies.logger.debug(error); + return []; + } + } + recall = async ({ user, queries, @@ -137,30 +182,42 @@ export class KnowledgeBaseService { () => `Recalling entries from KB for queries: "${JSON.stringify(queries)}"` ); - const [documentsFromKb, documentsFromConnectors] = await Promise.all([ - this.recallFromKnowledgeBase({ - user, - queries, - categories, - namespace, - }).catch((error) => { - if (isInferenceEndpointMissingOrUnavailable(error)) { - throwKnowledgeBaseNotReady(error); - } - throw error; - }), - recallFromSearchConnectors({ - esClient, - uiSettingsClient, - queries, - core: this.dependencies.core, - logger: this.dependencies.logger, - }).catch((error) => { - this.dependencies.logger.error('Error getting data from search indices'); - this.dependencies.logger.debug(error); - return []; - }), - ]); + const [documentsFromKb, documentsFromConnectors, documentsFromIntegrations] = await Promise.all( + [ + this.recallFromKnowledgeBase({ + user, + queries, + categories, + namespace, + }).catch((error) => { + if (isInferenceEndpointMissingOrUnavailable(error)) { + throwKnowledgeBaseNotReady(error); + } + throw error; + }), + recallFromSearchConnectors({ + esClient, + uiSettingsClient, + queries, + core: this.dependencies.core, + logger: this.dependencies.logger, + }).catch((error) => { + this.dependencies.logger.error('Error getting data from search indices'); + this.dependencies.logger.debug(error); + return []; + }), + this.recallFromIntegrationsKnowledge({ + esClient, + queries, + }).catch((error) => { + this.dependencies.logger.error( + `Error getting data from ${INTEGRATION_KNOWLEDGE_INDEX} index` + ); + this.dependencies.logger.debug(error); + return []; + }), + ] + ); this.dependencies.logger.debug( `documentsFromKb: ${JSON.stringify(documentsFromKb.slice(0, 5), null, 2)}` @@ -168,9 +225,12 @@ export class KnowledgeBaseService { this.dependencies.logger.debug( `documentsFromConnectors: ${JSON.stringify(documentsFromConnectors.slice(0, 5), null, 2)}` ); + this.dependencies.logger.debug( + `documentsFromIntegrations: ${JSON.stringify(documentsFromIntegrations.slice(0, 5), null, 2)}` + ); const sortedEntries = orderBy( - documentsFromKb.concat(documentsFromConnectors), + [...documentsFromKb, ...documentsFromConnectors, ...documentsFromIntegrations], 'esScore', 'desc' ).slice(0, limit.size ?? 20);