diff --git a/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js index 2b547d698415c..3eb369dec919d 100644 --- a/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js @@ -6,12 +6,18 @@ * Side Public License, v 1. */ -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; export class ComponentTemplateAutocompleteComponent extends ListComponent { constructor(name, parent) { - super(name, getAutocompleteInfo().getEntityProvider('componentTemplates'), parent, true, true); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.COMPONENT_TEMPLATES), + parent, + true, + true + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js index 0b043410c3b25..3ebc017c39796 100644 --- a/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js @@ -7,11 +7,16 @@ */ import { ListComponent } from './list_component'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; export class DataStreamAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('dataStreams'), parent, multiValued); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.DATA_STREAMS), + parent, + multiValued + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js index e3257b2bd86b8..3192bf830f4f3 100644 --- a/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js @@ -7,11 +7,11 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function FieldGenerator(context) { - return _.map(getAutocompleteInfo().getEntityProvider('fields', context), function (field) { + return _.map(getAutocompleteInfo().getEntityProvider(ENTITIES.FIELDS, context), function (field) { return { name: field.name, meta: field.type }; }); } diff --git a/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js index c2a7e2fb14286..7615c511148b0 100644 --- a/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js @@ -7,7 +7,7 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function nonValidIndexType(token) { @@ -16,7 +16,7 @@ function nonValidIndexType(token) { export class IndexAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued); + super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued); } validateTokens(tokens) { if (!this.multiValued && tokens.length > 1) { diff --git a/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js index 7bb3c32239751..41c760c809004 100644 --- a/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js @@ -6,12 +6,18 @@ * Side Public License, v 1. */ -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; export class IndexTemplateAutocompleteComponent extends ListComponent { constructor(name, parent) { - super(name, getAutocompleteInfo().getEntityProvider('indexTemplates'), parent, true, true); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.INDEX_TEMPLATES), + parent, + true, + true + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js index c505f66a68b0c..75394787533f1 100644 --- a/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js @@ -7,7 +7,7 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function nonValidUsernameType(token) { @@ -16,7 +16,7 @@ function nonValidUsernameType(token) { export class UsernameAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued); + super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued); } validateTokens(tokens) { if (!this.multiValued && tokens.length > 1) { diff --git a/src/plugins/console/public/lib/autocomplete_entities/types.ts b/src/plugins/console/public/lib/autocomplete_entities/types.ts index e49f8f106f37a..9c2607b8ad84f 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/types.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/types.ts @@ -29,7 +29,7 @@ export interface FieldMapping { fields?: FieldMapping[]; } -export interface MappingsApiResponse { +export interface AutoCompleteEntitiesApiResponse { mappings: IndicesGetMappingResponse; aliases: IndicesGetAliasResponse; dataStreams: IndicesGetDataStreamResponse; diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 59974184ea2f5..3e1a38a514607 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -8,7 +8,7 @@ import { createGetterSetter } from '@kbn/kibana-utils-plugin/public'; import type { HttpSetup } from '@kbn/core/public'; -import type { MappingsApiResponse } from '../lib/autocomplete_entities/types'; +import type { AutoCompleteEntitiesApiResponse } from '../lib/autocomplete_entities/types'; import { API_BASE_PATH } from '../../common/constants'; import { Alias, @@ -20,6 +20,15 @@ import { } from '../lib/autocomplete_entities'; import { DevToolsSettings, Settings } from './settings'; +export enum ENTITIES { + INDICES = 'indices', + FIELDS = 'fields', + INDEX_TEMPLATES = 'indexTemplates', + COMPONENT_TEMPLATES = 'componentTemplates', + LEGACY_TEMPLATES = 'legacyTemplates', + DATA_STREAMS = 'dataStreams', +} + export class AutocompleteInfo { public readonly alias = new Alias(); public readonly mapping = new Mapping(); @@ -39,19 +48,19 @@ export class AutocompleteInfo { context: { indices: string[]; types: string[] } = { indices: [], types: [] } ) { switch (type) { - case 'indices': + case ENTITIES.INDICES: const includeAliases = true; const collaborator = this.mapping; return () => this.alias.getIndices(includeAliases, collaborator); - case 'fields': + case ENTITIES.FIELDS: return this.mapping.getMappings(context.indices, context.types); - case 'indexTemplates': + case ENTITIES.INDEX_TEMPLATES: return () => this.indexTemplate.getTemplates(); - case 'componentTemplates': + case ENTITIES.COMPONENT_TEMPLATES: return () => this.componentTemplate.getTemplates(); - case 'legacyTemplates': + case ENTITIES.LEGACY_TEMPLATES: return () => this.legacyTemplate.getTemplates(); - case 'dataStreams': + case ENTITIES.DATA_STREAMS: return () => this.dataStream.getDataStreams(); default: throw new Error(`Unsupported type: ${type}`); @@ -61,7 +70,7 @@ export class AutocompleteInfo { public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) { this.clearSubscriptions(); this.http - .get(`${API_BASE_PATH}/autocomplete_entities`, { + .get(`${API_BASE_PATH}/autocomplete_entities`, { query: { ...settingsToRetrieve }, }) .then((data) => { @@ -83,7 +92,7 @@ export class AutocompleteInfo { } } - private load(data: MappingsApiResponse) { + private load(data: AutoCompleteEntitiesApiResponse) { this.mapping.loadMappings(data.mappings); const collaborator = this.mapping; this.alias.loadAliases(data.aliases, collaborator); diff --git a/src/plugins/console/public/services/index.ts b/src/plugins/console/public/services/index.ts index d73c169fd647a..73929f89e386f 100644 --- a/src/plugins/console/public/services/index.ts +++ b/src/plugins/console/public/services/index.ts @@ -10,4 +10,9 @@ export { createHistory, History } from './history'; export { createStorage, Storage, StorageKeys, setStorage, getStorage } from './storage'; export type { DevToolsSettings } from './settings'; export { createSettings, Settings, DEFAULT_SETTINGS } from './settings'; -export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo } from './autocomplete'; +export { + AutocompleteInfo, + getAutocompleteInfo, + setAutocompleteInfo, + ENTITIES, +} from './autocomplete'; diff --git a/src/plugins/console/server/lib/proxy_request.ts b/src/plugins/console/server/lib/proxy_request.ts index 875345be52055..90d3cafa8c566 100644 --- a/src/plugins/console/server/lib/proxy_request.ts +++ b/src/plugins/console/server/lib/proxy_request.ts @@ -12,6 +12,7 @@ import net from 'net'; import stream from 'stream'; import Boom from '@hapi/boom'; import { URL } from 'url'; +import { sanitizeHostname } from './utils'; interface Args { method: 'get' | 'post' | 'put' | 'delete' | 'patch' | 'head'; @@ -23,13 +24,6 @@ interface Args { rejectUnauthorized?: boolean; } -/** - * Node http request library does not expect there to be trailing "[" or "]" - * characters in ipv6 host names. - */ -const sanitizeHostname = (hostName: string): string => - hostName.trim().replace(/^\[/, '').replace(/\]$/, ''); - // We use a modified version of Hapi's Wreck because Hapi, Axios, and Superagent don't support GET requests // with bodies, but ES APIs do. Similarly with DELETE requests with bodies. Another library, `request` // diverged too much from current behaviour. diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index 2c595640eefbf..aac0f890a8a80 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -8,4 +8,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; -export { streamToJSON } from './stream_to_json'; +export { sanitizeHostname } from './sanitize_hostname'; diff --git a/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts b/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts new file mode 100644 index 0000000000000..987481ddfa5fd --- /dev/null +++ b/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { sanitizeHostname } from './sanitize_hostname'; + +describe('sanitizeHostname', () => { + it('should remove leading and trailing brackets', () => { + expect(sanitizeHostname('[::1]')).toBe('::1'); + }); + + it('should remove leading brackets', () => { + expect(sanitizeHostname('[::1')).toBe('::1'); + }); + + it('should remove trailing brackets', () => { + expect(sanitizeHostname('::1]')).toBe('::1'); + }); + + it('should not remove brackets in the middle of the string', () => { + expect(sanitizeHostname('[::1]foo')).toBe('::1]foo'); + }); +}); diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts b/src/plugins/console/server/lib/utils/sanitize_hostname.ts similarity index 59% rename from src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts rename to src/plugins/console/server/lib/utils/sanitize_hostname.ts index 53d12f69d30e5..a7b49c5874f58 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts +++ b/src/plugins/console/server/lib/utils/sanitize_hostname.ts @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -import type { RouteDependencies } from '../../..'; -import { registerGetRoute } from './register_get_route'; - -export function registerMappingsRoute(deps: RouteDependencies) { - registerGetRoute(deps); -} +/** + * Node http request library does not expect there to be trailing "[" or "]" + * characters in ipv6 host names. + */ +export const sanitizeHostname = (hostName: string): string => + hostName.trim().replace(/^\[/, '').replace(/\]$/, ''); diff --git a/src/plugins/console/server/lib/utils/stream_to_json.test.ts b/src/plugins/console/server/lib/utils/stream_to_json.test.ts deleted file mode 100644 index 780d3adb4a145..0000000000000 --- a/src/plugins/console/server/lib/utils/stream_to_json.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { Readable } from 'stream'; -import { streamToJSON } from './stream_to_json'; -import type { IncomingMessage } from 'http'; - -describe('streamToString', () => { - it('should limit the response size', async () => { - const stream = new Readable({ - read() { - this.push('a'.repeat(1000)); - }, - }); - await expect( - streamToJSON(stream as IncomingMessage, 500) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`); - }); - - it('should parse the response', async () => { - const stream = new Readable({ - read() { - this.push('{"test": "test"}'); - this.push(null); - }, - }); - const result = await streamToJSON(stream as IncomingMessage, 5000); - expect(result).toEqual({ test: 'test' }); - }); -}); diff --git a/src/plugins/console/server/lib/utils/stream_to_json.ts b/src/plugins/console/server/lib/utils/stream_to_json.ts deleted file mode 100644 index 5ff1974e9fd47..0000000000000 --- a/src/plugins/console/server/lib/utils/stream_to_json.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { IncomingMessage } from 'http'; - -export function streamToJSON(stream: IncomingMessage, limit: number) { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on('data', (chunk) => { - chunks.push(chunk); - if (Buffer.byteLength(Buffer.concat(chunks)) > limit) { - stream.destroy(); - reject(new Error('Response size limit exceeded')); - } - }); - stream.on('end', () => { - const response = Buffer.concat(chunks).toString('utf8'); - resolve(JSON.parse(response)); - }); - stream.on('error', reject); - }); -} diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index 796451b2721f3..1c6b23fa7fd7f 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -6,4 +6,207 @@ * Side Public License, v 1. */ -export { registerMappingsRoute } from './register_mappings_route'; +import http from 'http'; +import https from 'https'; +import { Buffer } from 'buffer'; +import { parse } from 'query-string'; +import Boom from '@hapi/boom'; +import type { KibanaRequest } from '@kbn/core-http-server'; +import type { SemVer } from 'semver'; +import type { RouteDependencies } from '../../..'; +import { sanitizeHostname } from '../../../../lib/utils'; +import type { ESConfigForProxy } from '../../../../types'; +import { getRequestConfig } from '../proxy/create_handler'; + +interface SettingsToRetrieve { + indices: boolean; + fields: boolean; + templates: boolean; + dataStreams: boolean; +} + +type Config = ESConfigForProxy & { headers: KibanaRequest['headers'] } & { kibanaVersion: SemVer }; + +const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB +// Limit the response size to 10MB, because the response can be very large and sending it to the client +// can cause the browser to hang. + +const getMappings = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.fields) { + const mappings = await getEntity('/_mapping', config); + return mappings; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +const getAliases = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.indices) { + const aliases = await getEntity('/_alias', config); + return aliases; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +const getDataStreams = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.dataStreams) { + const dataStreams = await getEntity('/_data_stream', config); + return dataStreams; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +const getLegacyTemplates = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.templates) { + const legacyTemplates = await getEntity('/_template', config); + return legacyTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +const getIndexTemplates = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.templates) { + const indexTemplates = await getEntity('/_index_template', config); + return indexTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +const getComponentTemplates = async (settings: SettingsToRetrieve, config: Config) => { + if (settings.templates) { + const componentTemplates = await getEntity('/_component_template', config); + return componentTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +}; + +/** + * Get the autocomplete suggestions for the given entity. + * We are using the raw http request in this function to retrieve the entities instead of esClient because + * the esClient does not handle large responses well. For example, the response size for + * the mappings can be very large(> 1GB) and the esClient will throw an 'Invalid string length' + * error when trying to parse the response. By using the raw http request, we can limit the + * response size and avoid the error. + * @param path The path to the entity to retrieve. For example, '/_mapping' or '/_alias'. + * @param config The configuration for the request. + * @returns The entity retrieved from Elasticsearch. + */ +const getEntity = (path: string, config: Config) => { + return new Promise((resolve, reject) => { + const { hosts, kibanaVersion } = config; + for (let idx = 0; idx < hosts.length; idx++) { + const host = hosts[idx]; + const uri = new URL(host + path); + const { protocol, hostname, port } = uri; + const { headers } = getRequestConfig(config.headers, config, uri.toString(), kibanaVersion); + const client = protocol === 'https:' ? https : http; + const options = { + method: 'GET', + headers: { ...headers }, + host: sanitizeHostname(hostname), + port: port === '' ? undefined : parseInt(port, 10), + protocol, + path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace + }; + + try { + const req = client.request(options, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + + // Destroy the request if the response is too large + if (Buffer.byteLength(Buffer.concat(chunks)) > MAX_RESPONSE_SIZE) { + req.destroy(); + reject(Boom.badRequest(`Response size is too large for ${path}`)); + } + }); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + resolve(JSON.parse(body)); + }); + }); + req.on('error', reject); + req.end(); + break; + } catch (err) { + if (idx === hosts.length - 1) { + reject(err); + } + // Try the next host + } + } + }); +}; + +export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { + deps.router.get( + { + path: '/api/console/autocomplete_entities', + options: { + tags: ['access:console'], + }, + validate: false, + }, + async (context, request, response) => { + const settings = parse(request.url.search, { + parseBooleans: true, + }) as unknown as SettingsToRetrieve; + + // If no settings are specified, then return 400. + if (Object.keys(settings).length === 0) { + return response.badRequest({ + body: 'Request must contain at least one of the following parameters: indices, fields, templates, dataStreams', + }); + } + + const legacyConfig = await deps.proxy.readLegacyESConfig(); + const configWithHeaders = { + ...legacyConfig, + headers: request.headers, + kibanaVersion: deps.kibanaVersion, + }; + + // Wait for all requests to complete, in case one of them fails return the successfull ones + const results = await Promise.allSettled([ + getMappings(settings, configWithHeaders), + getAliases(settings, configWithHeaders), + getDataStreams(settings, configWithHeaders), + getLegacyTemplates(settings, configWithHeaders), + getIndexTemplates(settings, configWithHeaders), + getComponentTemplates(settings, configWithHeaders), + ]); + + const [mappings, aliases, dataStreams, legacyTemplates, indexTemplates, componentTemplates] = + results.map((result) => { + // If the request was successful, return the result + if (result.status === 'fulfilled') { + return result.value; + } + + // If the request failed, log the error and return an empty object + if (result.reason instanceof Error) { + deps.log.debug(`Failed to retrieve autocomplete suggestions: ${result.reason.message}`); + } + + return {}; + }); + + return response.ok({ + body: { + mappings, + aliases, + dataStreams, + legacyTemplates, + indexTemplates, + componentTemplates, + }, + }); + } + ); +}; diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts deleted file mode 100644 index 98bd51e901b09..0000000000000 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import type { IScopedClusterClient } from '@kbn/core/server'; -import { parse } from 'query-string'; -import type { IncomingMessage } from 'http'; -import type { RouteDependencies } from '../../..'; -import { API_BASE_PATH } from '../../../../../common/constants'; -import { streamToJSON } from '../../../../lib/utils'; - -interface Settings { - indices: boolean; - fields: boolean; - templates: boolean; - dataStreams: boolean; -} - -const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; -// Limit the response size to 10MB, because the response can be very large and sending it to the client -// can cause the browser to hang. - -async function getMappings(esClient: IScopedClusterClient, settings: Settings) { - if (settings.fields) { - const stream = await esClient.asInternalUser.indices.getMapping(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getAliases(esClient: IScopedClusterClient, settings: Settings) { - if (settings.indices) { - const stream = await esClient.asInternalUser.indices.getAlias(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) { - if (settings.dataStreams) { - const stream = await esClient.asInternalUser.indices.getDataStream(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.indices.getTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getComponentTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getIndexTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -export function registerGetRoute({ router, lib: { handleEsError } }: RouteDependencies) { - router.get( - { - path: `${API_BASE_PATH}/autocomplete_entities`, - validate: false, - }, - async (ctx, request, response) => { - try { - const settings = parse(request.url.search, { parseBooleans: true }) as unknown as Settings; - - // If no settings are provided return 400 - if (Object.keys(settings).length === 0) { - return response.badRequest({ - body: 'Request must contain a query param of autocomplete settings', - }); - } - - const esClient = (await ctx.core).elasticsearch.client; - - // Wait for all requests to complete, in case one of them fails return the successfull ones - const results = await Promise.allSettled([ - getMappings(esClient, settings), - getAliases(esClient, settings), - getDataStreams(esClient, settings), - getLegacyTemplates(esClient, settings), - getIndexTemplates(esClient, settings), - getComponentTemplates(esClient, settings), - ]); - - const [ - mappings, - aliases, - dataStreams, - legacyTemplates, - indexTemplates, - componentTemplates, - ] = results.map((result) => { - // If the request was successful, return the result - if (result.status === 'fulfilled') { - return result.value; - } - // If the request failed, return an empty object - return {}; - }); - - return response.ok({ - body: { - mappings, - aliases, - dataStreams, - legacyTemplates, - indexTemplates, - componentTemplates, - }, - }); - } catch (e) { - return handleEsError({ error: e, response }); - } - } - ); -} diff --git a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts index eddefc86fcbd2..70de8d6c59c7f 100644 --- a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts +++ b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts @@ -43,7 +43,7 @@ function filterHeaders(originalHeaders: object, headersToKeep: string[]): object return pick(originalHeaders, headersToKeepNormalized); } -function getRequestConfig( +export function getRequestConfig( headers: object, esConfig: ESConfigForProxy, uri: string, diff --git a/src/plugins/console/server/routes/index.ts b/src/plugins/console/server/routes/index.ts index b82b2ffbffa8e..e1a036f55a62c 100644 --- a/src/plugins/console/server/routes/index.ts +++ b/src/plugins/console/server/routes/index.ts @@ -17,7 +17,7 @@ import { handleEsError } from '../shared_imports'; import { registerEsConfigRoute } from './api/console/es_config'; import { registerProxyRoute } from './api/console/proxy'; import { registerSpecDefinitionsRoute } from './api/console/spec_definitions'; -import { registerMappingsRoute } from './api/console/autocomplete_entities'; +import { registerAutocompleteEntitiesRoute } from './api/console/autocomplete_entities'; export interface ProxyDependencies { readLegacyESConfig: () => Promise; @@ -43,5 +43,5 @@ export const registerRoutes = (dependencies: RouteDependencies) => { registerEsConfigRoute(dependencies); registerProxyRoute(dependencies); registerSpecDefinitionsRoute(dependencies); - registerMappingsRoute(dependencies); + registerAutocompleteEntitiesRoute(dependencies); }; diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts index 7f74156f379a0..6bd899c979a2b 100644 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -7,127 +7,275 @@ */ import expect from '@kbn/expect'; -import type { Response } from 'superagent'; import type { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const client = getService('es'); - function utilTest(name: string, query: object, test: (response: Response) => void) { - it(name, async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query(query); - test(response); + const createIndex = async (indexName: string) => { + await client.indices.create({ + index: indexName, + body: { + mappings: { + properties: { + foo: { + type: 'text', + }, + }, + }, + }, + }); + }; + + const createAlias = async (indexName: string, aliasName: string) => { + await client.indices.putAlias({ + index: indexName, + name: aliasName, + }); + }; + + const createLegacyTemplate = async (templateName: string) => { + await client.indices.putTemplate({ + name: templateName, + body: { + index_patterns: ['*'], + }, + }); + }; + + const createComponentTemplate = async (templateName: string) => { + await client.cluster.putComponentTemplate({ + name: templateName, + body: { + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + format: 'date_optional_time||epoch_millis', + }, + message: { + type: 'wildcard', + }, + }, + }, + }, + }, + }); + }; + + const createIndexTemplate = async ( + templateName: string, + indexPatterns: string[], + composedOf: string[] + ) => { + await client.indices.putIndexTemplate({ + name: templateName, + body: { + index_patterns: indexPatterns, + data_stream: {}, + composed_of: composedOf, + priority: 500, + }, + }); + }; + + const createDataStream = async (dataStream: string) => { + await client.indices.createDataStream({ + name: dataStream, + }); + }; + + const deleteIndex = async (indexName: string) => { + await client.indices.delete({ + index: indexName, + }); + }; + + const deleteAlias = async (indexName: string, aliasName: string) => { + await client.indices.deleteAlias({ + index: indexName, + name: aliasName, + }); + }; + + const deleteIndexTemplate = async (templateName: string) => { + await client.indices.deleteIndexTemplate({ + name: templateName, + }); + }; + + const deleteComponentTemplate = async (templateName: string) => { + await client.cluster.deleteComponentTemplate({ + name: templateName, + }); + }; + + const deleteLegacyTemplate = async (templateName: string) => { + await client.indices.deleteTemplate({ + name: templateName, }); - } + }; + + const deleteDataStream = async (dataStream: string) => { + await client.indices.deleteDataStream({ + name: dataStream, + }); + }; + + const sendRequest = async (query: object) => { + return await supertest.get('/api/console/autocomplete_entities').query(query); + }; describe('/api/console/autocomplete_entities', () => { - utilTest('should not succeed if no settings are provided in query params', {}, (response) => { + const indexName = 'test-index-1'; + const aliasName = 'test-alias-1'; + const indexTemplateName = 'test-index-template-1'; + const componentTemplateName = 'test-component-template-1'; + const dataStreamName = 'test-data-stream-1'; + const legacyTemplateName = 'test-legacy-template-1'; + + before(async () => { + // Setup indices, aliases, templates, and data streams + await createIndex(indexName); + await createAlias(indexName, aliasName); + await createComponentTemplate(componentTemplateName); + await createIndexTemplate(indexTemplateName, [dataStreamName], [componentTemplateName]); + await createDataStream(dataStreamName); + await createLegacyTemplate(legacyTemplateName); + }); + + after(async () => { + // Cleanup indices, aliases, templates, and data streams + await deleteAlias(indexName, aliasName); + await deleteIndex(indexName); + await deleteDataStream(dataStreamName); + await deleteIndexTemplate(indexTemplateName); + await deleteComponentTemplate(componentTemplateName); + await deleteLegacyTemplate(legacyTemplateName); + }); + + it('should not succeed if no settings are provided in query params', async () => { + const response = await sendRequest({}); const { status } = response; expect(status).to.be(400); }); - utilTest( - 'should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', - { + it('should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', async () => { + const response = await sendRequest({ indices: true, fields: true, templates: true, dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(Object.keys(body).sort()).to.eql([ - 'aliases', - 'componentTemplates', - 'dataStreams', - 'indexTemplates', - 'legacyTemplates', - 'mappings', - ]); - } - ); - - utilTest( - 'should return empty payload with all settings are set to false', - { + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body).sort()).to.eql([ + 'aliases', + 'componentTemplates', + 'dataStreams', + 'indexTemplates', + 'legacyTemplates', + 'mappings', + ]); + }); + + it('should return empty payload with all settings are set to false', async () => { + const response = await sendRequest({ indices: false, fields: false, templates: false, dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - expect(body.aliases).to.eql({}); - expect(body.mappings).to.eql({}); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty templates with templates setting is set to false', - { - indices: true, - fields: true, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + expect(body.aliases).to.eql({}); + expect(body.mappings).to.eql({}); + expect(body.dataStreams).to.eql({}); + }); + + it('should return empty templates with templates setting is set to false', async () => { + const response = await sendRequest({ templates: false, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - } - ); - - utilTest( - 'should return empty data streams with dataStreams setting is set to false', - { - indices: true, - fields: true, - templates: true, + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + }); + + it('should return empty data streams with dataStreams setting is set to false', async () => { + const response = await sendRequest({ dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty aliases with indices setting is set to false', - { + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.dataStreams).to.eql({}); + }); + + it('should return empty aliases with indices setting is set to false', async () => { + const response = await sendRequest({ indices: false, - fields: true, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.aliases).to.eql({}); - } - ); - - utilTest( - 'should return empty mappings with fields setting is set to false', - { - indices: true, + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.aliases).to.eql({}); + }); + + it('should return empty mappings with fields setting is set to false', async () => { + const response = await sendRequest({ fields: false, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.mappings).to.eql({}); - } - ); + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.mappings).to.eql({}); + }); + + it('should return mappings with fields setting is set to true', async () => { + const response = await sendRequest({ fields: true }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body.mappings)).to.contain(indexName); + }); + + it('should return aliases with indices setting is set to true', async () => { + const response = await sendRequest({ indices: true }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.aliases[indexName].aliases).to.eql({ [aliasName]: {} }); + }); + + it('should return data streams with dataStreams setting is set to true', async () => { + const response = await sendRequest({ dataStreams: true }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.dataStreams.data_streams.map((ds: { name: string }) => ds.name)).to.contain( + dataStreamName + ); + }); + + it('should return all templates with templates setting is set to true', async () => { + const response = await sendRequest({ templates: true }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body.legacyTemplates)).to.contain(legacyTemplateName); + expect(body.indexTemplates.index_templates.map((it: { name: string }) => it.name)).to.contain( + indexTemplateName + ); + expect( + body.componentTemplates.component_templates.map((ct: { name: string }) => ct.name) + ).to.contain(componentTemplateName); + }); }); };