[AI Infra] Fix Observability AI assistant product docs missing multilingual support

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -915,6 +915,7 @@
   ],
   "product-doc-install-status": [
     "index_name",
+    "inference_id",
     "installation_status",
     "last_installation_date",
     "product_name",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -3042,6 +3042,9 @@
       "index_name": {
         "type": "keyword"
       },
+      "inference_id": {
+        "type": "keyword"
+      },
       "installation_status": {
         "type": "keyword"
       },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -154,7 +154,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "osquery-saved-query": "a8ef11610473e3d1b51a8fdacb2799d8a610818e",
         "policy-settings-protection-updates-note": "c05c4c33a5e5bd1fa153991f300d040ac5d6f38d",
         "privilege-monitoring-status": "4daec76df427409bcd64250f5c23f5ab86c8bac3",
-        "product-doc-install-status": "ee7817c45bf1c41830290c8ef535e726c86f7c19",
+        "product-doc-install-status": "f94e3e5ad2cc933df918f2cd159044c626e01011",
         "query": "1966ccce8e9853018111fb8a1dee500228731d9e",
         "risk-engine-configuration": "533a0a3f2dbef1c95129146ec4d5714de305be1a",
         "rules-settings": "53f94e5ce61f5e75d55ab8adbc1fb3d0937d2e0b",

x-pack/packages/ai-infra/product-doc-artifact-builder/README.md

@@ -8,17 +8,23 @@ Script to build the knowledge base artifacts.
 node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product}
 ```
 
+Example:
+
+```
+node scripts/build_product_doc_artifacts.js --product-name=security --stack-version=8.18  --inference-id=.multilingual-e5-small-elasticsearch
+```
+
 ### parameters
 
-#### `stack-version`: 
+#### `stack-version`:
 
 the stack version to generate the artifacts for.
 
-#### `product-name`: 
+#### `product-name`:
 
 (multi-value) the list of products to generate artifacts for.
 
-possible values: 
+possible values:
 - "kibana"
 - "elasticsearch"
 - "observability"
@@ -34,6 +40,11 @@ Defaults to `{REPO_ROOT}/build-kb-artifacts`.
 
 The folder to use for temporary files.
 
+#### inference-id:
+
+The inference endpoint to use to generate the embeddings. If the inference ID provided and is not the ELSER default, the artifacts will be generated with `{artifactName}--{inference-id}.zip`. Note the double dash before inference-id.
+
+
 Defaults to `{REPO_ROOT}/build/temp-kb-artifacts`
 
 #### Cluster infos
@@ -46,4 +57,7 @@ Defaults to `{REPO_ROOT}/build/temp-kb-artifacts`
 - params for the embedding cluster:
 `embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL
 `embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME
-`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD
+`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD
+
+- params for the inference endpoint:
+`inferenceId`

x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts

@@ -6,30 +6,31 @@
  */
 
 import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import {
+  DEFAULT_ELSER,
+  getSemanticTextMapping,
+  type SemanticTextMapping,
+} from '../tasks/create_index';
 
-export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMapping => {
+export const getArtifactMappings = (
+  customSemanticTextMapping?: SemanticTextMapping
+): MappingTypeMapping => {
+  const semanticTextMapping = customSemanticTextMapping
+    ? customSemanticTextMapping
+    : getSemanticTextMapping(DEFAULT_ELSER);
   return {
     dynamic: 'strict',
     properties: {
       content_title: { type: 'text' },
-      content_body: {
-        type: 'semantic_text',
-        inference_id: inferenceEndpoint,
-      },
+      content_body: semanticTextMapping,
       product_name: { type: 'keyword' },
       root_type: { type: 'keyword' },
       slug: { type: 'keyword' },
       url: { type: 'keyword' },
       version: { type: 'version' },
       ai_subtitle: { type: 'text' },
-      ai_summary: {
-        type: 'semantic_text',
-        inference_id: inferenceEndpoint,
-      },
-      ai_questions_answered: {
-        type: 'semantic_text',
-        inference_id: inferenceEndpoint,
-      },
+      ai_summary: semanticTextMapping,
+      ai_questions_answered: semanticTextMapping,
       ai_tags: { type: 'keyword' },
     },
   };

x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts

@@ -7,8 +7,14 @@
 
 import Path from 'path';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
+import {
+  Client as ElasticsearchClient8,
+  HttpConnection as Elasticsearch8HttpConnection,
+} from 'elasticsearch-8.x';
+
 import { ToolingLog } from '@kbn/tooling-log';
 import type { ProductName } from '@kbn/product-doc-common';
+import { defaultInferenceEndpoints } from '@kbn/inference-common';
 import {
   // checkConnectivity,
   createTargetIndex,
@@ -21,17 +27,18 @@ import {
   processDocuments,
 } from './tasks';
 import type { TaskConfig } from './types';
+import { getSemanticTextMapping } from './tasks/create_index';
 
 const getSourceClient = (config: TaskConfig) => {
-  return new Client({
+  return new ElasticsearchClient8({
     compression: true,
     nodes: [config.sourceClusterUrl],
     sniffOnStart: false,
     auth: {
       username: config.sourceClusterUsername,
       password: config.sourceClusterPassword,
     },
-    Connection: HttpConnection,
+    Connection: Elasticsearch8HttpConnection,
     requestTimeout: 30_000,
   });
 };
@@ -79,6 +86,7 @@ export const buildArtifacts = async (config: TaskConfig) => {
       sourceClient,
       embeddingClient,
       log,
+      inferenceId: config.inferenceId ?? defaultInferenceEndpoints.ELSER,
     });
   }
 
@@ -93,18 +101,41 @@ const buildArtifact = async ({
   embeddingClient,
   sourceClient,
   log,
+  inferenceId,
 }: {
   productName: ProductName;
   stackVersion: string;
   buildFolder: string;
   targetFolder: string;
-  sourceClient: Client;
+  sourceClient: ElasticsearchClient8;
   embeddingClient: Client;
   log: ToolingLog;
+  inferenceId: string;
 }) => {
-  log.info(`Starting building artifact for product [${productName}] and version [${stackVersion}]`);
+  log.info(
+    `Starting building artifact for product [${productName}] and version [${stackVersion}] with inference id [${inferenceId}]`
+  );
 
-  const targetIndex = getTargetIndexName({ productName, stackVersion });
+  const semanticTextMapping = getSemanticTextMapping(inferenceId);
+
+  log.info(
+    `Detected semantic text mapping for Inference ID ${inferenceId}:\n ${JSON.stringify(
+      semanticTextMapping,
+      null,
+      2
+    )}`
+  );
+
+  const targetIndex = getTargetIndexName({
+    productName,
+    stackVersion,
+    inferenceId: semanticTextMapping?.inference_id,
+  });
+  await deleteIndex({
+    indexName: targetIndex,
+    client: embeddingClient,
+    log,
+  });
 
   let documents = await extractDocumentation({
     client: sourceClient,
@@ -119,6 +150,7 @@ const buildArtifact = async ({
   await createTargetIndex({
     client: embeddingClient,
     indexName: targetIndex,
+    semanticTextMapping,
   });
 
   await indexDocuments({
@@ -142,12 +174,7 @@ const buildArtifact = async ({
     productName,
     stackVersion,
     log,
-  });
-
-  await deleteIndex({
-    indexName: targetIndex,
-    client: embeddingClient,
-    log,
+    semanticTextMapping,
   });
 
   log.info(`Finished building artifact for product [${productName}] and version [${stackVersion}]`);
@@ -156,9 +183,13 @@ const buildArtifact = async ({
 const getTargetIndexName = ({
   productName,
   stackVersion,
+  inferenceId,
 }: {
   productName: string;
   stackVersion: string;
+  inferenceId?: string;
 }) => {
-  return `kb-artifact-builder-${productName}-${stackVersion}`.toLowerCase();
+  return `kb-artifact-builder-${productName}-${stackVersion}${
+    inferenceId ? `-${inferenceId}` : ''
+  }`.toLowerCase();
 };

x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts

@@ -71,6 +71,10 @@ function options(y: yargs.Argv) {
       demandOption: true,
       default: process.env.KIBANA_EMBEDDING_CLUSTER_PASSWORD,
     })
+    .option('inferenceId', {
+      describe: 'The inference id to use for the artifacts',
+      string: true,
+    })
     .locale('en');
 }
 
@@ -89,6 +93,7 @@ export function runScript() {
         embeddingClusterUrl: argv.embeddingClusterUrl!,
         embeddingClusterUsername: argv.embeddingClusterUsername!,
         embeddingClusterPassword: argv.embeddingClusterPassword!,
+        inferenceId: argv.inferenceId,
       };
 
       return buildArtifacts(taskConfig);

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts

@@ -15,28 +15,32 @@ import {
 } from '@kbn/product-doc-common';
 import { getArtifactMappings } from '../artifact/mappings';
 import { getArtifactManifest } from '../artifact/manifest';
-import { DEFAULT_ELSER } from './create_index';
+import { DEFAULT_ELSER, SemanticTextMapping } from './create_index';
 
 export const createArtifact = async ({
   productName,
   stackVersion,
   buildFolder,
   targetFolder,
   log,
+  semanticTextMapping,
 }: {
   buildFolder: string;
   targetFolder: string;
   productName: ProductName;
   stackVersion: string;
   log: ToolingLog;
+  semanticTextMapping?: SemanticTextMapping;
 }) => {
   log.info(
     `Starting to create artifact from build folder [${buildFolder}] into target [${targetFolder}]`
   );
 
   const zip = new AdmZip();
 
-  const mappings = getArtifactMappings(DEFAULT_ELSER);
+  const inferenceId = semanticTextMapping?.inference_id || DEFAULT_ELSER;
+
+  const mappings = getArtifactMappings(semanticTextMapping);
   const mappingFileContent = JSON.stringify(mappings, undefined, 2);
   zip.addFile('mappings.json', Buffer.from(mappingFileContent, 'utf-8'));
 
@@ -53,6 +57,7 @@ export const createArtifact = async ({
   const artifactName = getArtifactName({
     productName,
     productVersion: stackVersion,
+    inferenceId,
   });
   zip.writeZip(Path.join(targetFolder, artifactName));
 

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts

@@ -6,43 +6,66 @@
  */
 
 import type { Client } from '@elastic/elasticsearch';
-import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import { getArtifactMappings } from '../artifact/mappings';
 
 export const DEFAULT_ELSER = '.elser-2-elasticsearch';
+export const DEFAULT_E5_SMALL = '.multilingual-e5-small-elasticsearch';
 
-const mappings: MappingTypeMapping = {
-  dynamic: 'strict',
-  properties: {
-    content_title: { type: 'text' },
-    content_body: {
-      type: 'semantic_text',
-      inference_id: DEFAULT_ELSER,
-    },
-    product_name: { type: 'keyword' },
-    root_type: { type: 'keyword' },
-    slug: { type: 'keyword' },
-    url: { type: 'keyword' },
-    version: { type: 'version' },
-    ai_subtitle: { type: 'text' },
-    ai_summary: {
-      type: 'semantic_text',
-      inference_id: DEFAULT_ELSER,
-    },
-    ai_questions_answered: {
-      type: 'semantic_text',
-      inference_id: DEFAULT_ELSER,
+interface BaseSemanticTextMapping {
+  type: 'semantic_text';
+  inference_id: string;
+}
+export interface SemanticTextMapping extends BaseSemanticTextMapping {
+  model_settings?: {
+    service?: string;
+    task_type?: string;
+    dimensions?: number;
+    similarity?: string;
+    element_type?: string;
+  };
+}
+
+type SupportedInferenceId = typeof DEFAULT_E5_SMALL | typeof DEFAULT_ELSER;
+const isSupportedInferenceId = (inferenceId: string): inferenceId is SupportedInferenceId => {
+  return inferenceId === DEFAULT_E5_SMALL || inferenceId === DEFAULT_ELSER;
+};
+
+const INFERENCE_ID_TO_SEMANTIC_TEXT_MAPPING: Record<SupportedInferenceId, SemanticTextMapping> = {
+  [DEFAULT_E5_SMALL]: {
+    type: 'semantic_text',
+    inference_id: DEFAULT_E5_SMALL,
+    model_settings: {
+      service: 'elasticsearch',
+      task_type: 'text_embedding',
+      dimensions: 384,
+      similarity: 'cosine',
+      element_type: 'float',
     },
-    ai_tags: { type: 'keyword' },
   },
+  [DEFAULT_ELSER]: {
+    type: 'semantic_text',
+    inference_id: DEFAULT_ELSER,
+  },
+};
+export const getSemanticTextMapping = (inferenceId: string): SemanticTextMapping => {
+  if (isSupportedInferenceId(inferenceId)) {
+    return INFERENCE_ID_TO_SEMANTIC_TEXT_MAPPING[inferenceId];
+  }
+  throw new Error(`Semantic text mapping for Inference ID ${inferenceId} not found`);
 };
 
 export const createTargetIndex = async ({
   indexName,
   client,
+  semanticTextMapping,
 }: {
   indexName: string;
   client: Client;
+  semanticTextMapping?: SemanticTextMapping;
 }) => {
+  const mappings = semanticTextMapping
+    ? getArtifactMappings(semanticTextMapping)
+    : getArtifactMappings(getSemanticTextMapping(DEFAULT_ELSER));
   await client.indices.create({
     index: indexName,
     mappings,