diff --git a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx index 4a696cb90f861..91ff156be0a24 100644 --- a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx +++ b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx @@ -432,6 +432,7 @@ export const KnowledgeBaseSettingsManagement: React.FC = React.memo(({ d entry={selectedEntry as IndexEntry} originalEntry={originalEntry as IndexEntry} dataViews={dataViews} + http={http} setEntry={ setSelectedEntry as React.Dispatch>> } diff --git a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.test.tsx b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.test.tsx index b0dba7faffdbe..68f61a3b0679f 100644 --- a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.test.tsx +++ b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.test.tsx @@ -6,13 +6,16 @@ */ import React from 'react'; -import userEvent from '@testing-library/user-event'; import { render, fireEvent, waitFor, within } from '@testing-library/react'; import { IndexEntryEditor } from './index_entry_editor'; import type { DataViewsContract } from '@kbn/data-views-plugin/public'; import type { IndexEntry } from '@kbn/elastic-assistant-common'; import * as i18n from './translations'; import { I18nProvider } from '@kbn/i18n-react'; +import { useIndexMappings } from './use_index_mappings'; +import type { HttpSetup } from '@kbn/core-http-browser'; + +jest.mock('./use_index_mappings'); const Wrapper = ({ children }: { children?: React.ReactNode }) => ( {children} @@ -21,17 +24,15 @@ const Wrapper = ({ children }: { children?: React.ReactNode }) => ( describe('IndexEntryEditor', () => { const mockSetEntry = jest.fn(); const mockDataViews = { - getFieldsForWildcard: jest.fn().mockResolvedValue([ - { name: 'field-1', esTypes: ['semantic_text'] }, - { name: 'field-2', esTypes: ['text'] }, - { name: 'field-3', esTypes: ['semantic_text'] }, - ]), getExistingIndices: jest.fn().mockResolvedValue(['index-1']), getIndices: jest.fn().mockResolvedValue([ { name: 'index-1', attributes: ['open'] }, { name: 'index-2', attributes: ['open'] }, ]), } as unknown as DataViewsContract; + const http = { + get: jest.fn(), + } as unknown as HttpSetup; const defaultProps = { dataViews: mockDataViews, @@ -46,10 +47,38 @@ describe('IndexEntryEditor', () => { queryDescription: 'Test Query Description', users: [], } as unknown as IndexEntry, + http, }; beforeEach(() => { jest.clearAllMocks(); + (useIndexMappings as jest.Mock).mockImplementation(({ indexName }) => { + if (indexName === 'index-1') { + return { + data: { + mappings: { + properties: { + 'field-1-text': { type: 'text' }, + 'field-1-keyword': { type: 'keyword' }, + }, + }, + }, + }; + } + if (indexName === 'index-2') { + return { + data: { + mappings: { + properties: { + 'field-2-text': { type: 'text' }, + 'field-2-semantic': { type: 'semantic_text' }, + }, + }, + }, + }; + } + return { data: undefined }; + }); }); it('renders the form fields with initial values', async () => { @@ -131,31 +160,50 @@ describe('IndexEntryEditor', () => { }); it('fetches field options based on selected index and updates on selection', async () => { - const { getByTestId, getAllByTestId } = render(, { - wrapper: Wrapper, - }); + const { getByTestId, queryByTestId, rerender } = render( + , + { + wrapper: Wrapper, + } + ); await waitFor(() => { - expect(mockDataViews.getFieldsForWildcard).toHaveBeenCalledWith({ - pattern: 'index-1', + expect(useIndexMappings).toHaveBeenCalledWith({ + http, + indexName: 'index-1', }); }); - await waitFor(async () => { - fireEvent.click(getByTestId('index-combobox')); - fireEvent.click(getAllByTestId('comboBoxToggleListButton')[0]); + // Open field dropdown and check options for index-1 + fireEvent.click(within(getByTestId('entry-combobox')).getByTestId('comboBoxToggleListButton')); + await waitFor(() => { + expect(getByTestId('field-option-field-1-text')).toBeInTheDocument(); + expect(queryByTestId('field-option-field-1-keyword')).not.toBeInTheDocument(); }); - fireEvent.click(getByTestId('index-2')); + + // Close the dropdown before re-rendering + fireEvent.click(within(getByTestId('entry-combobox')).getByTestId('comboBoxToggleListButton')); + + // Change index to index-2 + const newEntry = { ...defaultProps.entry, index: 'index-2', field: '' }; + rerender(); await waitFor(() => { - fireEvent.click(getByTestId('entry-combobox')); + expect(useIndexMappings).toHaveBeenCalledWith({ + http, + indexName: 'index-2', + }); }); - await userEvent.type( - within(getByTestId('entry-combobox')).getByTestId('comboBoxSearchInput'), - 'field-3' - ); + // Open field dropdown and check options for index-2 + fireEvent.click(within(getByTestId('entry-combobox')).getByTestId('comboBoxToggleListButton')); + await waitFor(() => { + expect(getByTestId('field-option-field-2-text')).toBeInTheDocument(); + expect(getByTestId('field-option-field-2-semantic')).toBeInTheDocument(); + }); + // Select a new field + fireEvent.click(getByTestId('field-option-field-2-text')); await waitFor(() => { expect(mockSetEntry).toHaveBeenCalledWith(expect.any(Function)); }); diff --git a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx index dc02fbee39842..1d566488935c7 100644 --- a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx +++ b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx @@ -7,6 +7,7 @@ import type { EuiComboBoxOptionOption } from '@elastic/eui'; import { + EuiBadge, EuiComboBox, EuiFieldText, EuiForm, @@ -22,10 +23,13 @@ import React, { useCallback, useMemo } from 'react'; import type { IndexEntry } from '@kbn/elastic-assistant-common'; import type { DataViewsContract } from '@kbn/data-views-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; +import type { HttpSetup } from '@kbn/core-http-browser'; +import { useIndexMappings } from './use_index_mappings'; import * as i18n from './translations'; import { isGlobalEntry } from './helpers'; interface Props { + http: HttpSetup; dataViews: DataViewsContract; entry?: IndexEntry; originalEntry?: IndexEntry; @@ -35,7 +39,7 @@ interface Props { } export const IndexEntryEditor: React.FC = React.memo( - ({ dataViews, entry, setEntry, hasManageGlobalKnowledgeBase, originalEntry, docLink }) => { + ({ http, dataViews, entry, setEntry, hasManageGlobalKnowledgeBase, originalEntry, docLink }) => { const privateUsers = useMemo(() => { const originalUsers = originalEntry?.users; if (originalEntry && !isGlobalEntry(originalEntry)) { @@ -122,36 +126,33 @@ export const IndexEntryEditor: React.FC = React.memo( return !(await dataViews.getExistingIndices([entry.index])).length; }, [entry?.index]); - const indexFields = useAsync( - async () => - dataViews.getFieldsForWildcard({ - pattern: entry?.index ?? '', - }), - [entry?.index] - ); + const { data: mappingData } = useIndexMappings({ + http, + indexName: entry?.index ?? '', + }); const fieldOptions = useMemo( () => - Array.isArray(indexFields?.value) - ? indexFields.value - .filter((field) => field.esTypes?.includes('text')) - .map((field) => ({ - 'data-test-subj': field.name, - label: field.name, - value: field.name, - })) - : [], - [indexFields?.value] + Object.entries(mappingData?.mappings?.properties ?? {}) + .filter(([, m]) => m.type === 'text' || m.type === 'semantic_text') + .map(([name, details]) => ({ + 'data-test-subj': `field-option-${name}`, + label: name, + value: name, + append: {details.type}, + })), + [mappingData] ); const outputFieldOptions = useMemo( () => - indexFields?.value?.map((field) => ({ - 'data-test-subj': field.name, - label: field.name, - value: field.name, - })) ?? [], - [indexFields?.value] + Object.entries(mappingData?.mappings?.properties ?? {}).map(([name, details]) => ({ + 'data-test-subj': `output-field-option-${name}`, + label: name, + value: name, + append: {details.type}, + })), + [mappingData] ); const onCreateIndexOption = (searchValue: string) => { diff --git a/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/use_index_mappings.ts b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/use_index_mappings.ts new file mode 100644 index 0000000000000..db021a32dd043 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/use_index_mappings.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryResult } from '@tanstack/react-query'; +import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; +import type { IToasts } from '@kbn/core-notifications-browser'; +import { i18n } from '@kbn/i18n'; +import type { MappingPropertyBase } from '@elastic/elasticsearch/lib/api/types'; +import { useQuery } from '@tanstack/react-query'; + +const KNOWLEDGE_BASE_MAPPINGS_QUERY_KEY = ['elastic-assistant', 'knowledge-base-mappings']; + +export interface Mappings { + mappings: { + properties: MappingPropertyBase['properties']; + }; +} + +export interface UseIndexMappingParams { + http: HttpSetup; + indexName: string; + toasts?: IToasts; +} + +/** + * Hook for getting index mappings + * + * @param {Object} options - The options object. + * @param {HttpSetup} [options.http] - HttpSetup + * @param {String} [options.indexName] - String + * @param {IToasts} [options.toasts] - IToasts + * + * @returns {useQuery} hook for getting mappings for a given index + */ +export const useIndexMappings = ({ + indexName, + http, + toasts, +}: UseIndexMappingParams): UseQueryResult => { + return useQuery( + [KNOWLEDGE_BASE_MAPPINGS_QUERY_KEY, indexName], + async ({ signal }) => { + return http.fetch( + `/api/index_management/mapping/${encodeURIComponent(indexName)}`, + { signal } + ); + }, + { + enabled: !!indexName, + onError: (error: IHttpFetchError) => { + if (error.name !== 'AbortError') { + toasts?.addError( + error.body && error.body.message ? new Error(error.body.message) : error, + { + title: i18n.translate('xpack.elasticAssistant.knowledgeBase.mappingsError', { + defaultMessage: 'Error fetching Knowledge Base mappings', + }), + } + ); + } + }, + } + ); +}; diff --git a/x-pack/solutions/security/plugins/elastic_assistant/package.json b/x-pack/solutions/security/plugins/elastic_assistant/package.json index 43015a8e9a145..8e67cd912d63b 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/package.json +++ b/x-pack/solutions/security/plugins/elastic_assistant/package.json @@ -8,6 +8,7 @@ "evaluate-model": "node ./scripts/model_evaluator", "generate-security-ai-prompts": "node ./scripts/generate_security_ai_prompts", "draw-graph": "node ./scripts/draw_graph", - "encode-security-labs-content": "node ./scripts/encode_security_labs_content" + "encode-security-labs-content": "node ./scripts/encode_security_labs_content", + "stress-test-mappings": "node ./scripts/stess_test_mappings" } } diff --git a/x-pack/solutions/security/plugins/elastic_assistant/scripts/stess_test_mappings.js b/x-pack/solutions/security/plugins/elastic_assistant/scripts/stess_test_mappings.js new file mode 100644 index 0000000000000..2d83b832961b1 --- /dev/null +++ b/x-pack/solutions/security/plugins/elastic_assistant/scripts/stess_test_mappings.js @@ -0,0 +1,337 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable no-process-exit */ + +/** + * Stress Test Mappings Script -- generated almost entirely by gemini-pro-2.5 via gemini-cli. + */ + +require('../../../../../../src/setup_node_env'); +const http = require('http'); +const https = require('https'); +const readline = require('readline'); + +function parseArgs() { + const args = process.argv.slice(2).reduce((acc, arg, i, arr) => { + if (arg.startsWith('--')) { + const key = arg.slice(2); + const next = arr[i + 1]; + if (next && !next.startsWith('--')) { + acc[key] = next; + } else { + acc[key] = true; + } + } + return acc; + }, {}); + + if (args.help || args.h) { + console.log(` + Elasticsearch Index/Mapping Populator and Cleanup Script + + Usage: + node stress_test_mappings.js [options] + node stress_test_mappings.js --cleanup + node stress_test_mappings.js --delete-by-count + + Description: + This script stress-tests an Elasticsearch instance by creating a large number + of indices with many fields. It can also clean up the indices it creates. + + Creation Options: + --host Elasticsearch host URL (default: http://localhost:9200) + --user Username for basic auth (default: elastic) + --pass Password for basic auth (default: changeme) + --apiKey API key for authentication (overrides user/pass) + --indices Number of indices to create (default: 5000) + --mappings Number of mappings per index (default: 5000) + --maxFields The max number of fields per index (default: same as --mappings) + --shards Number of primary shards per index (default: 1) + --replicas Number of replicas per index (default: 0) + + Cleanup & Recovery Options: + --cleanup Delete all indices created by this script. + --delete-by-count Delete the newest stress-test indices. + --yes Bypass confirmation prompt during cleanup. + + Other Options: + -h, --help Show this help message + `); + process.exit(0); + } + + return { + host: args.host || 'http://localhost:9200', + user: args.user || 'elastic', + pass: args.pass || 'changeme', + apiKey: args.apiKey, + indices: parseInt(args.indices, 10) || 5000, + mappings: parseInt(args.mappings, 10) || 5000, + maxFields: parseInt(args.maxFields, 10) || parseInt(args.mappings, 10) || 5000, + shards: parseInt(args.shards, 10) || 1, + replicas: parseInt(args.replicas, 10) || 0, + cleanup: !!args.cleanup, + deleteByCount: parseInt(args['delete-by-count'], 10) || 0, + yes: !!args.yes, + }; +} + +const config = parseArgs(); + +const simpleFieldTypes = [ + { type: 'text' }, + { type: 'keyword' }, + { type: 'long' }, + { type: 'integer' }, + { type: 'short' }, + { type: 'byte' }, + { type: 'double' }, + { type: 'float' }, + { type: 'half_float' }, + { type: 'scaled_float', scaling_factor: 100 }, + { type: 'date' }, + { type: 'date_nanos' }, + { type: 'boolean' }, + { type: 'binary' }, + { type: 'geo_point' }, + { type: 'ip' }, + { type: 'completion' }, + { type: 'token_count', analyzer: 'standard' }, +]; + +const complexFieldTypes = [ + { type: 'integer_range' }, + { type: 'float_range' }, + { type: 'long_range' }, + { type: 'double_range' }, + { type: 'date_range' }, + { type: 'geo_shape' }, + { type: 'search_as_you_type' }, + { type: 'dense_vector', dims: 4 }, + { type: 'semantic_text' }, +]; + +function generateIndexBody(numMappings, maxFields, numShards, numReplicas) { + const properties = {}; + let fieldCount = 0; + + for (const fieldType of complexFieldTypes) { + if (fieldCount >= numMappings) break; + properties[`complex_${fieldType.type}_${fieldCount}`] = { ...fieldType }; + fieldCount++; + } + + while (fieldCount < numMappings) { + const fieldTypeDefinition = simpleFieldTypes[fieldCount % simpleFieldTypes.length]; + properties[`field_${fieldCount}`] = { ...fieldTypeDefinition }; + fieldCount++; + } + + return { + settings: { + 'index.mapping.total_fields.limit': maxFields, + 'index.number_of_shards': numShards, + 'index.number_of_replicas': numReplicas, + }, + mappings: { properties }, + }; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function makeRequest(method, path, body, retries = 3, delay = 1000) { + for (let i = 0; i < retries; i++) { + try { + return await new Promise((resolve, reject) => { + const url = new URL(config.host); + const protocol = url.protocol === 'https:' ? https : http; + const options = { + hostname: url.hostname, + port: url.port, + path, + method, + headers: { 'Content-Type': 'application/json' }, + }; + + if (config.apiKey) { + options.headers.Authorization = `ApiKey ${config.apiKey}`; + } else if (config.user && config.pass) { + const auth = `Basic ${Buffer.from(`${config.user}:${config.pass}`).toString('base64')}`; + options.headers.Authorization = auth; + } + + const req = protocol.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + resolve({ statusCode: res.statusCode, body: JSON.parse(data || '{}') }); + } catch (e) { + reject(new Error('Failed to parse JSON response.')); + } + } else { + const err = new Error(`Request failed with status code ${res.statusCode}: ${data}`); + if (data.includes('resource_already_exists_exception')) { + err.isAlreadyExists = true; + } + if ([429, 503, 504].includes(res.statusCode)) { + err.isRetryable = true; + } + reject(err); + } + }); + }); + + req.on('error', (e) => reject(e)); + if (body) req.write(JSON.stringify(body)); + req.end(); + }); + } catch (error) { + if (error.isAlreadyExists || !error.isRetryable || i === retries - 1) { + throw error; + } + await sleep(delay); + // eslint-disable-next-line no-param-reassign + delay *= 2; + } + } +} + +async function cleanupIndices() { + console.log('Starting cleanup of stress-test indices...'); + if (!config.yes) { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + await new Promise((resolve) => { + rl.question( + 'Are you sure you want to delete all indices with the pattern "*-stress-test-index-*"? (y/N) ', + (answer) => { + if (answer.toLowerCase() !== 'y') { + console.log('Cleanup cancelled.'); + process.exit(0); + } + rl.close(); + resolve(); + } + ); + }); + } + + try { + const { body } = await makeRequest('DELETE', '/*-stress-test-index-*'); + console.log('Cleanup successful:', body); + } catch (error) { + if (error.message.includes('404')) { + console.log('No stress-test indices found to delete.'); + } else { + console.error('An error occurred during cleanup:', error.message); + process.exit(1); + } + } +} + +async function deleteIndicesByCount(count) { + console.log(`Fetching the ${count} newest stress-test indices to delete...`); + try { + const { body } = await makeRequest( + 'GET', + `/_cat/indices/*-stress-test-index-*?h=index&s=creation.date:desc&format=json` + ); + const indices = body.map((item) => item.index); + + if (indices.length === 0) { + console.log('No stress-test indices found to delete.'); + return; + } + + const batchToDelete = indices.slice(0, count); + console.log(`Deleting ${batchToDelete.length} indices: ${batchToDelete.join(', ')}`); + await makeRequest('DELETE', `/${batchToDelete.join(',')}`); + console.log('Deletion successful.'); + } catch (e) { + console.error('\n[FATAL] Could not get or delete indices:', e.message); + process.exit(1); + } +} + +async function createIndices() { + console.log('Starting to populate Elasticsearch...'); + console.log('Configuration:', { + ...config, + pass: '***', + apiKey: config.apiKey ? '***' : undefined, + }); + + const alphabet = 'abcdefghijklmnopqrstuvwxyz'; + let createdCount = 0; + let skippedCount = 0; + const total = config.indices; + const barWidth = 40; + + for (let i = 0; i < total; i++) { + const indexName = `${alphabet[i % alphabet.length]}-stress-test-index-${String(i).padStart( + 5, + '0' + )}`; + const percent = (i + 1) / total; + const filledWidth = Math.round(barWidth * percent); + const bar = `[${'█'.repeat(filledWidth)}${'-'.repeat(barWidth - filledWidth)}]`; + const percentStr = `${(percent * 100).toFixed(1)}%`; + + readline.clearLine(process.stdout, 0); + readline.cursorTo(process.stdout, 0); + process.stdout.write(`${bar} ${percentStr} | [${i + 1}/${total}] Processing: ${indexName}`); + + const indexBody = generateIndexBody( + config.mappings, + config.maxFields, + config.shards, + config.replicas + ); + + try { + await makeRequest('PUT', `/${indexName}`, indexBody); + createdCount++; + } catch (error) { + if (error.isAlreadyExists) { + skippedCount++; + // eslint-disable-next-line no-continue + continue; + } + + process.stdout.write('\n'); + console.error(`\n[FATAL] Failed while processing index ${indexName}:`, error.message); + console.error( + 'Exiting due to a critical error. Please check your Elasticsearch cluster status and settings.' + ); + + process.exit(1); + } + } + process.stdout.write('\n'); + console.log( + `\nPopulation complete. Created: ${createdCount}, Skipped: ${skippedCount}, Total processed: ${ + createdCount + skippedCount + }.` + ); +} + +async function main() { + if (config.cleanup) { + await cleanupIndices(); + } else if (config.deleteByCount > 0) { + await deleteIndicesByCount(config.deleteByCount); + } else { + await createIndices(); + } +} + +main().catch((err) => { + console.error('\nAn unexpected error occurred:', err.message); + process.exit(1); +});