From c6c304df623800706b98c4165a9e72949c177d30 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Wed, 5 Oct 2022 11:06:50 +0500 Subject: [PATCH 01/17] Fix loading autocomplete suggestions on client-side --- ...mponent_template_autocomplete_component.js | 10 +- .../data_stream_autocomplete_component.js | 9 +- .../field_autocomplete_component.js | 4 +- .../index_autocomplete_component.js | 4 +- .../index_template_autocomplete_component.js | 10 +- .../username_autocomplete_component.js | 4 +- .../public/lib/autocomplete_entities/types.ts | 18 -- src/plugins/console/public/plugin.ts | 2 +- .../console/public/services/autocomplete.ts | 202 ++++++++++++++---- src/plugins/console/public/services/index.ts | 2 +- .../console/autocomplete_entities/index.ts | 9 - .../register_get_route.ts | 152 ------------- .../register_mappings_route.ts | 14 -- src/plugins/console/server/routes/index.ts | 2 - 14 files changed, 197 insertions(+), 245 deletions(-) delete mode 100644 src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts delete mode 100644 src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts delete mode 100644 src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts diff --git a/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js index 2b547d698415c..3eb369dec919d 100644 --- a/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js @@ -6,12 +6,18 @@ * Side Public License, v 1. */ -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; export class ComponentTemplateAutocompleteComponent extends ListComponent { constructor(name, parent) { - super(name, getAutocompleteInfo().getEntityProvider('componentTemplates'), parent, true, true); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.COMPONENT_TEMPLATES), + parent, + true, + true + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js index 0b043410c3b25..3ebc017c39796 100644 --- a/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js @@ -7,11 +7,16 @@ */ import { ListComponent } from './list_component'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; export class DataStreamAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('dataStreams'), parent, multiValued); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.DATA_STREAMS), + parent, + multiValued + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js index e3257b2bd86b8..3192bf830f4f3 100644 --- a/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js @@ -7,11 +7,11 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function FieldGenerator(context) { - return _.map(getAutocompleteInfo().getEntityProvider('fields', context), function (field) { + return _.map(getAutocompleteInfo().getEntityProvider(ENTITIES.FIELDS, context), function (field) { return { name: field.name, meta: field.type }; }); } diff --git a/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js index c2a7e2fb14286..7615c511148b0 100644 --- a/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js @@ -7,7 +7,7 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function nonValidIndexType(token) { @@ -16,7 +16,7 @@ function nonValidIndexType(token) { export class IndexAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued); + super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued); } validateTokens(tokens) { if (!this.multiValued && tokens.length > 1) { diff --git a/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js index 7bb3c32239751..41c760c809004 100644 --- a/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js @@ -6,12 +6,18 @@ * Side Public License, v 1. */ -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; export class IndexTemplateAutocompleteComponent extends ListComponent { constructor(name, parent) { - super(name, getAutocompleteInfo().getEntityProvider('indexTemplates'), parent, true, true); + super( + name, + getAutocompleteInfo().getEntityProvider(ENTITIES.INDEX_TEMPLATES), + parent, + true, + true + ); } getContextKey() { diff --git a/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js b/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js index c505f66a68b0c..75394787533f1 100644 --- a/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js +++ b/src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js @@ -7,7 +7,7 @@ */ import _ from 'lodash'; -import { getAutocompleteInfo } from '../../../services'; +import { getAutocompleteInfo, ENTITIES } from '../../../services'; import { ListComponent } from './list_component'; function nonValidUsernameType(token) { @@ -16,7 +16,7 @@ function nonValidUsernameType(token) { export class UsernameAutocompleteComponent extends ListComponent { constructor(name, parent, multiValued) { - super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued); + super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued); } validateTokens(tokens) { if (!this.multiValued && tokens.length > 1) { diff --git a/src/plugins/console/public/lib/autocomplete_entities/types.ts b/src/plugins/console/public/lib/autocomplete_entities/types.ts index e49f8f106f37a..38a280193b61c 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/types.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/types.ts @@ -6,15 +6,6 @@ * Side Public License, v 1. */ -import type { - ClusterGetComponentTemplateResponse, - IndicesGetAliasResponse, - IndicesGetDataStreamResponse, - IndicesGetIndexTemplateResponse, - IndicesGetMappingResponse, - IndicesGetTemplateResponse, -} from '@elastic/elasticsearch/lib/api/types'; - export interface Field { name: string; type: string; @@ -28,12 +19,3 @@ export interface FieldMapping { index_name?: string; fields?: FieldMapping[]; } - -export interface MappingsApiResponse { - mappings: IndicesGetMappingResponse; - aliases: IndicesGetAliasResponse; - dataStreams: IndicesGetDataStreamResponse; - legacyTemplates: IndicesGetTemplateResponse; - indexTemplates: IndicesGetIndexTemplateResponse; - componentTemplates: ClusterGetComponentTemplateResponse; -} diff --git a/src/plugins/console/public/plugin.ts b/src/plugins/console/public/plugin.ts index 33ee5446dc268..886759e2dc896 100644 --- a/src/plugins/console/public/plugin.ts +++ b/src/plugins/console/public/plugin.ts @@ -29,7 +29,7 @@ export class ConsoleUIPlugin implements Plugin(); - this.autocompleteInfo.setup(http); + this.autocompleteInfo.setup(http, notifications); setAutocompleteInfo(this.autocompleteInfo); if (isConsoleUiEnabled) { diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 59974184ea2f5..436ee6ae7c592 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -7,9 +7,19 @@ */ import { createGetterSetter } from '@kbn/kibana-utils-plugin/public'; +import { i18n } from '@kbn/i18n'; + import type { HttpSetup } from '@kbn/core/public'; -import type { MappingsApiResponse } from '../lib/autocomplete_entities/types'; -import { API_BASE_PATH } from '../../common/constants'; +import type { + ClusterGetComponentTemplateResponse, + IndicesGetAliasResponse, + IndicesGetDataStreamResponse, + IndicesGetIndexTemplateResponse, + IndicesGetMappingResponse, + IndicesGetTemplateResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import type { NotificationsSetup } from '@kbn/core-notifications-browser'; +import { send } from '../lib/es'; import { Alias, DataStream, @@ -18,7 +28,23 @@ import { IndexTemplate, ComponentTemplate, } from '../lib/autocomplete_entities'; -import { DevToolsSettings, Settings } from './settings'; +import type { DevToolsSettings, Settings } from './settings'; + +export enum ENTITIES { + INDICES = 'indices', + FIELDS = 'fields', + INDEX_TEMPLATES = 'indexTemplates', + COMPONENT_TEMPLATES = 'componentTemplates', + LEGACY_TEMPLATES = 'legacyTemplates', + DATA_STREAMS = 'dataStreams', +} + +type SettingsToRetrieve = DevToolsSettings['autocomplete'] & { + indexTemplates: boolean; + componentTemplates: boolean; + legacyTemplates: boolean; +}; +type SettingsKey = keyof Omit; export class AutocompleteInfo { public readonly alias = new Alias(); @@ -28,10 +54,12 @@ export class AutocompleteInfo { public readonly indexTemplate = new IndexTemplate(); public readonly componentTemplate = new ComponentTemplate(); private http!: HttpSetup; + private notifications!: NotificationsSetup; private pollTimeoutId: ReturnType | undefined; - public setup(http: HttpSetup) { + public setup(http: HttpSetup, notifications: NotificationsSetup) { this.http = http; + this.notifications = notifications; } public getEntityProvider( @@ -39,42 +67,154 @@ export class AutocompleteInfo { context: { indices: string[]; types: string[] } = { indices: [], types: [] } ) { switch (type) { - case 'indices': + case ENTITIES.INDICES: const includeAliases = true; const collaborator = this.mapping; return () => this.alias.getIndices(includeAliases, collaborator); - case 'fields': + case ENTITIES.FIELDS: return this.mapping.getMappings(context.indices, context.types); - case 'indexTemplates': + case ENTITIES.INDEX_TEMPLATES: return () => this.indexTemplate.getTemplates(); - case 'componentTemplates': + case ENTITIES.COMPONENT_TEMPLATES: return () => this.componentTemplate.getTemplates(); - case 'legacyTemplates': + case ENTITIES.LEGACY_TEMPLATES: return () => this.legacyTemplate.getTemplates(); - case 'dataStreams': + case ENTITIES.DATA_STREAMS: return () => this.dataStream.getDataStreams(); default: - throw new Error(`Unsupported type: ${type}`); + throw new Error(`Unknown entity type: ${type}`); } } - public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) { + public retrieve(settings: Settings, settingsToRetrieve: SettingsToRetrieve) { this.clearSubscriptions(); - this.http - .get(`${API_BASE_PATH}/autocomplete_entities`, { - query: { ...settingsToRetrieve }, - }) - .then((data) => { - this.load(data); - // Schedule next request. - this.pollTimeoutId = setTimeout(() => { - // This looks strange/inefficient, but it ensures correct behavior because we don't want to send - // a scheduled request if the user turns off polling. - if (settings.getPolling()) { - this.retrieve(settings, settings.getAutocomplete()); - } - }, settings.getPollInterval()); + const templateSettingsToRetrieve = { + ...settingsToRetrieve, + legacyTemplates: settingsToRetrieve.templates, + indexTemplates: settingsToRetrieve.templates, + componentTemplates: settingsToRetrieve.templates, + }; + + Promise.allSettled([ + this.retrieveMappings(settingsToRetrieve), + this.retrieveAliases(settingsToRetrieve), + this.retrieveDataStreams(settingsToRetrieve), + this.retrieveTemplates(templateSettingsToRetrieve), + ]).then(() => { + this.pollTimeoutId = setTimeout(() => { + // This looks strange/inefficient, but it ensures correct behavior because we don't want to send + // a scheduled request if the user turns off polling. + if (settings.getPolling()) { + this.retrieve(settings, settings.getAutocomplete()); + } + }, settings.getPollInterval()); + }); + } + + // Move fetching autocomplete to client-side to avoid browser event loop blocking + + private retrieveSettings( + settingsKey: SettingsKey, + settingsToRetrieve: SettingsToRetrieve + ): Promise { + const settingKeyToPathMap = { + fields: '_mapping', + indices: '_aliases', + legacyTemplates: '_template', + indexTemplates: '_index_template', + componentTemplates: '_component_template', + dataStreams: '_data_stream', + }; + + // Fetch autocomplete info if setting is enabled and if user has made changes. + if (settingsToRetrieve[settingsKey]) { + return send({ + http: this.http, + method: 'GET', + path: `${settingKeyToPathMap[settingsKey]}?pretty=false`, // pretty=false to compress the response and save bandwidth by avoiding whitespace + asSystemRequest: true, }); + } else { + // If the user doesn't want autocomplete suggestions, then clear any that exist + return Promise.resolve({}); + } + } + + private async retrieveMappings(settingsToRetrieve: SettingsToRetrieve) { + const mappings = await this.retrieveSettings( + ENTITIES.FIELDS, + settingsToRetrieve + ); + + if (mappings) { + const maxMappingSize = Object.keys(mappings).length > 10 * 1024 * 1024; + let mappingsResponse; + + if (maxMappingSize) { + this.notifications.toasts.addWarning({ + title: i18n.translate('console.autocomplete.mappingsTooLargeTitle', { + defaultMessage: 'Mappings too large', + }), + text: i18n.translate('console.autocomplete.mappingsTooLargeText', { + defaultMessage: 'Mappings are too large to be displayed in autocomplete.', + }), + }); + mappingsResponse = {}; + } else { + mappingsResponse = mappings; + } + this.mapping.loadMappings(mappingsResponse); + } + } + + private async retrieveAliases(settingsToRetrieve: SettingsToRetrieve) { + const aliases = await this.retrieveSettings( + ENTITIES.INDICES, + settingsToRetrieve + ); + + if (aliases) { + const collaborator = this.mapping; + this.alias.loadAliases(aliases, collaborator); + } + } + + private async retrieveDataStreams(settingsToRetrieve: SettingsToRetrieve) { + const dataStreams = await this.retrieveSettings( + ENTITIES.DATA_STREAMS, + settingsToRetrieve + ); + + if (dataStreams && 'data_streams' in dataStreams) { + this.dataStream.loadDataStreams(dataStreams); + } + } + + private async retrieveTemplates(settingsToRetrieve: SettingsToRetrieve) { + const legacyTemplates = await this.retrieveSettings( + ENTITIES.LEGACY_TEMPLATES, + settingsToRetrieve + ); + const indexTemplates = await this.retrieveSettings( + ENTITIES.INDEX_TEMPLATES, + settingsToRetrieve + ); + const componentTemplates = await this.retrieveSettings( + ENTITIES.COMPONENT_TEMPLATES, + settingsToRetrieve + ); + + if (legacyTemplates) { + this.legacyTemplate.loadTemplates(legacyTemplates); + } + + if (indexTemplates && 'index_templates' in indexTemplates) { + this.indexTemplate.loadTemplates(indexTemplates); + } + + if (componentTemplates && 'component_templates' in componentTemplates) { + this.componentTemplate.loadTemplates(componentTemplates); + } } public clearSubscriptions() { @@ -83,16 +223,6 @@ export class AutocompleteInfo { } } - private load(data: MappingsApiResponse) { - this.mapping.loadMappings(data.mappings); - const collaborator = this.mapping; - this.alias.loadAliases(data.aliases, collaborator); - this.indexTemplate.loadTemplates(data.indexTemplates); - this.componentTemplate.loadTemplates(data.componentTemplates); - this.legacyTemplate.loadTemplates(data.legacyTemplates); - this.dataStream.loadDataStreams(data.dataStreams); - } - public clear() { this.alias.clearAliases(); this.mapping.clearMappings(); diff --git a/src/plugins/console/public/services/index.ts b/src/plugins/console/public/services/index.ts index d73c169fd647a..f15c8f94cc970 100644 --- a/src/plugins/console/public/services/index.ts +++ b/src/plugins/console/public/services/index.ts @@ -10,4 +10,4 @@ export { createHistory, History } from './history'; export { createStorage, Storage, StorageKeys, setStorage, getStorage } from './storage'; export type { DevToolsSettings } from './settings'; export { createSettings, Settings, DEFAULT_SETTINGS } from './settings'; -export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo } from './autocomplete'; +export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo, ENTITIES } from './autocomplete'; diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts deleted file mode 100644 index 796451b2721f3..0000000000000 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export { registerMappingsRoute } from './register_mappings_route'; diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts deleted file mode 100644 index 98bd51e901b09..0000000000000 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import type { IScopedClusterClient } from '@kbn/core/server'; -import { parse } from 'query-string'; -import type { IncomingMessage } from 'http'; -import type { RouteDependencies } from '../../..'; -import { API_BASE_PATH } from '../../../../../common/constants'; -import { streamToJSON } from '../../../../lib/utils'; - -interface Settings { - indices: boolean; - fields: boolean; - templates: boolean; - dataStreams: boolean; -} - -const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; -// Limit the response size to 10MB, because the response can be very large and sending it to the client -// can cause the browser to hang. - -async function getMappings(esClient: IScopedClusterClient, settings: Settings) { - if (settings.fields) { - const stream = await esClient.asInternalUser.indices.getMapping(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getAliases(esClient: IScopedClusterClient, settings: Settings) { - if (settings.indices) { - const stream = await esClient.asInternalUser.indices.getAlias(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) { - if (settings.dataStreams) { - const stream = await esClient.asInternalUser.indices.getDataStream(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.indices.getTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getComponentTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -async function getIndexTemplates(esClient: IScopedClusterClient, settings: Settings) { - if (settings.templates) { - const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, { - asStream: true, - }); - return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - } - // If the user doesn't want autocomplete suggestions, then clear any that exist. - return {}; -} - -export function registerGetRoute({ router, lib: { handleEsError } }: RouteDependencies) { - router.get( - { - path: `${API_BASE_PATH}/autocomplete_entities`, - validate: false, - }, - async (ctx, request, response) => { - try { - const settings = parse(request.url.search, { parseBooleans: true }) as unknown as Settings; - - // If no settings are provided return 400 - if (Object.keys(settings).length === 0) { - return response.badRequest({ - body: 'Request must contain a query param of autocomplete settings', - }); - } - - const esClient = (await ctx.core).elasticsearch.client; - - // Wait for all requests to complete, in case one of them fails return the successfull ones - const results = await Promise.allSettled([ - getMappings(esClient, settings), - getAliases(esClient, settings), - getDataStreams(esClient, settings), - getLegacyTemplates(esClient, settings), - getIndexTemplates(esClient, settings), - getComponentTemplates(esClient, settings), - ]); - - const [ - mappings, - aliases, - dataStreams, - legacyTemplates, - indexTemplates, - componentTemplates, - ] = results.map((result) => { - // If the request was successful, return the result - if (result.status === 'fulfilled') { - return result.value; - } - // If the request failed, return an empty object - return {}; - }); - - return response.ok({ - body: { - mappings, - aliases, - dataStreams, - legacyTemplates, - indexTemplates, - componentTemplates, - }, - }); - } catch (e) { - return handleEsError({ error: e, response }); - } - } - ); -} diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts deleted file mode 100644 index 53d12f69d30e5..0000000000000 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_mappings_route.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { RouteDependencies } from '../../..'; -import { registerGetRoute } from './register_get_route'; - -export function registerMappingsRoute(deps: RouteDependencies) { - registerGetRoute(deps); -} diff --git a/src/plugins/console/server/routes/index.ts b/src/plugins/console/server/routes/index.ts index b82b2ffbffa8e..8554c6461818f 100644 --- a/src/plugins/console/server/routes/index.ts +++ b/src/plugins/console/server/routes/index.ts @@ -17,7 +17,6 @@ import { handleEsError } from '../shared_imports'; import { registerEsConfigRoute } from './api/console/es_config'; import { registerProxyRoute } from './api/console/proxy'; import { registerSpecDefinitionsRoute } from './api/console/spec_definitions'; -import { registerMappingsRoute } from './api/console/autocomplete_entities'; export interface ProxyDependencies { readLegacyESConfig: () => Promise; @@ -43,5 +42,4 @@ export const registerRoutes = (dependencies: RouteDependencies) => { registerEsConfigRoute(dependencies); registerProxyRoute(dependencies); registerSpecDefinitionsRoute(dependencies); - registerMappingsRoute(dependencies); }; From 6a34210636774c084efab55e1aec1f77fc05372b Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Wed, 5 Oct 2022 11:38:29 +0500 Subject: [PATCH 02/17] Remove unnecessary comment --- src/plugins/console/public/services/autocomplete.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 436ee6ae7c592..9fbd893c4878b 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -111,8 +111,6 @@ export class AutocompleteInfo { }); } - // Move fetching autocomplete to client-side to avoid browser event loop blocking - private retrieveSettings( settingsKey: SettingsKey, settingsToRetrieve: SettingsToRetrieve From 8a6e8362b1cca925c1ff9224a680d89e0b95200b Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Wed, 5 Oct 2022 11:44:56 +0500 Subject: [PATCH 03/17] Fix type checks --- .../console/public/application/containers/settings.tsx | 5 +++-- src/plugins/console/public/services/autocomplete.ts | 2 +- src/plugins/console/public/services/index.ts | 8 +++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/plugins/console/public/application/containers/settings.tsx b/src/plugins/console/public/application/containers/settings.tsx index b9a9d68294e6d..878f6dc64eb74 100644 --- a/src/plugins/console/public/application/containers/settings.tsx +++ b/src/plugins/console/public/application/containers/settings.tsx @@ -13,6 +13,7 @@ import { AutocompleteOptions, DevToolsSettingsModal } from '../components'; import { useServicesContext, useEditorActionContext } from '../contexts'; import { DevToolsSettings, Settings as SettingsService } from '../../services'; import type { SenseEditor } from '../models'; +import type { SettingsToRetrieve } from '../../services'; const getAutocompleteDiff = ( newSettings: DevToolsSettings, @@ -38,7 +39,7 @@ export function Settings({ onClose, editorInstance }: Props) { const refreshAutocompleteSettings = ( settingsService: SettingsService, - selectedSettings: DevToolsSettings['autocomplete'] + selectedSettings: SettingsToRetrieve ) => { autocompleteInfo.retrieve(settingsService, selectedSettings); }; @@ -99,7 +100,7 @@ export function Settings({ onClose, editorInstance }: Props) { onClose={onClose} onSaveSettings={onSaveSettings} refreshAutocompleteSettings={(selectedSettings) => - refreshAutocompleteSettings(settings, selectedSettings) + refreshAutocompleteSettings(settings, selectedSettings as SettingsToRetrieve) } settings={settings.toJSON()} editorInstance={editorInstance} diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 9fbd893c4878b..10fbe6ff9c993 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -39,7 +39,7 @@ export enum ENTITIES { DATA_STREAMS = 'dataStreams', } -type SettingsToRetrieve = DevToolsSettings['autocomplete'] & { +export type SettingsToRetrieve = DevToolsSettings['autocomplete'] & { indexTemplates: boolean; componentTemplates: boolean; legacyTemplates: boolean; diff --git a/src/plugins/console/public/services/index.ts b/src/plugins/console/public/services/index.ts index f15c8f94cc970..01d190d4e150e 100644 --- a/src/plugins/console/public/services/index.ts +++ b/src/plugins/console/public/services/index.ts @@ -10,4 +10,10 @@ export { createHistory, History } from './history'; export { createStorage, Storage, StorageKeys, setStorage, getStorage } from './storage'; export type { DevToolsSettings } from './settings'; export { createSettings, Settings, DEFAULT_SETTINGS } from './settings'; -export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo, ENTITIES } from './autocomplete'; +export { + AutocompleteInfo, + getAutocompleteInfo, + setAutocompleteInfo, + ENTITIES, +} from './autocomplete'; +export type { SettingsToRetrieve } from './autocomplete'; From 341f8392ecb2ffa632367e68a283ac94a55afc52 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Wed, 5 Oct 2022 13:39:23 +0500 Subject: [PATCH 04/17] Remove tests for deleted route --- .../apis/console/autocomplete_entities.ts | 133 ------------------ test/api_integration/apis/console/index.ts | 1 - 2 files changed, 134 deletions(-) delete mode 100644 test/api_integration/apis/console/autocomplete_entities.ts diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts deleted file mode 100644 index 7f74156f379a0..0000000000000 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import expect from '@kbn/expect'; -import type { Response } from 'superagent'; -import type { FtrProviderContext } from '../../ftr_provider_context'; - -export default ({ getService }: FtrProviderContext) => { - const supertest = getService('supertest'); - - function utilTest(name: string, query: object, test: (response: Response) => void) { - it(name, async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query(query); - test(response); - }); - } - - describe('/api/console/autocomplete_entities', () => { - utilTest('should not succeed if no settings are provided in query params', {}, (response) => { - const { status } = response; - expect(status).to.be(400); - }); - - utilTest( - 'should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', - { - indices: true, - fields: true, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(Object.keys(body).sort()).to.eql([ - 'aliases', - 'componentTemplates', - 'dataStreams', - 'indexTemplates', - 'legacyTemplates', - 'mappings', - ]); - } - ); - - utilTest( - 'should return empty payload with all settings are set to false', - { - indices: false, - fields: false, - templates: false, - dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - expect(body.aliases).to.eql({}); - expect(body.mappings).to.eql({}); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty templates with templates setting is set to false', - { - indices: true, - fields: true, - templates: false, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - } - ); - - utilTest( - 'should return empty data streams with dataStreams setting is set to false', - { - indices: true, - fields: true, - templates: true, - dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty aliases with indices setting is set to false', - { - indices: false, - fields: true, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.aliases).to.eql({}); - } - ); - - utilTest( - 'should return empty mappings with fields setting is set to false', - { - indices: true, - fields: false, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.mappings).to.eql({}); - } - ); - }); -}; diff --git a/test/api_integration/apis/console/index.ts b/test/api_integration/apis/console/index.ts index 81f6f17f77b87..ad4f8256f97ad 100644 --- a/test/api_integration/apis/console/index.ts +++ b/test/api_integration/apis/console/index.ts @@ -11,6 +11,5 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('core', () => { loadTestFile(require.resolve('./proxy_route')); - loadTestFile(require.resolve('./autocomplete_entities')); }); } From 4519c7c05b82c4ec070fdc73d7533d835ea4b774 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Fri, 7 Oct 2022 12:30:04 +0500 Subject: [PATCH 05/17] Address comments --- .../console/public/services/autocomplete.ts | 71 +++++++++++-------- .../console/server/lib/proxy_request.ts | 40 ++++++++++- 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 10fbe6ff9c993..e4263bd2750ce 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -100,7 +100,29 @@ export class AutocompleteInfo { this.retrieveAliases(settingsToRetrieve), this.retrieveDataStreams(settingsToRetrieve), this.retrieveTemplates(templateSettingsToRetrieve), - ]).then(() => { + ]).then((response) => { + const errors = response.filter((result) => result.status === 'rejected'); + if (errors.length) { + // Notify the user if mapping size is too large + const isMappingResponseExceededError = errors.some((error) => { + if ('reason' in error) { + return error.reason?.body?.message === 'Maximum size of mappings response exceeded'; + } + }); + + if (isMappingResponseExceededError) { + this.notifications.toasts.addWarning({ + title: i18n.translate('devTools.autocomplete.mappingResponseExceededTitle', { + defaultMessage: 'Maximum size of mappings response exceeded', + }), + text: i18n.translate('devTools.autocomplete.mappingResponseExceededText', { + defaultMessage: + 'Some autocomplete suggestions may be missing. Disable autocomplete suggestions for fields in Settings to optimize performance.', + }), + }); + } + } + this.pollTimeoutId = setTimeout(() => { // This looks strange/inefficient, but it ensures correct behavior because we don't want to send // a scheduled request if the user turns off polling. @@ -129,7 +151,8 @@ export class AutocompleteInfo { return send({ http: this.http, method: 'GET', - path: `${settingKeyToPathMap[settingsKey]}?pretty=false`, // pretty=false to compress the response and save bandwidth by avoiding whitespace + // pretty=false to compress the response and save bandwidth by avoiding whitespace + path: `${settingKeyToPathMap[settingsKey]}?pretty=false`, asSystemRequest: true, }); } else { @@ -145,23 +168,7 @@ export class AutocompleteInfo { ); if (mappings) { - const maxMappingSize = Object.keys(mappings).length > 10 * 1024 * 1024; - let mappingsResponse; - - if (maxMappingSize) { - this.notifications.toasts.addWarning({ - title: i18n.translate('console.autocomplete.mappingsTooLargeTitle', { - defaultMessage: 'Mappings too large', - }), - text: i18n.translate('console.autocomplete.mappingsTooLargeText', { - defaultMessage: 'Mappings are too large to be displayed in autocomplete.', - }), - }); - mappingsResponse = {}; - } else { - mappingsResponse = mappings; - } - this.mapping.loadMappings(mappingsResponse); + this.mapping.loadMappings(mappings); } } @@ -189,18 +196,20 @@ export class AutocompleteInfo { } private async retrieveTemplates(settingsToRetrieve: SettingsToRetrieve) { - const legacyTemplates = await this.retrieveSettings( - ENTITIES.LEGACY_TEMPLATES, - settingsToRetrieve - ); - const indexTemplates = await this.retrieveSettings( - ENTITIES.INDEX_TEMPLATES, - settingsToRetrieve - ); - const componentTemplates = await this.retrieveSettings( - ENTITIES.COMPONENT_TEMPLATES, - settingsToRetrieve - ); + const [legacyTemplates, indexTemplates, componentTemplates] = await Promise.all([ + this.retrieveSettings( + ENTITIES.LEGACY_TEMPLATES, + settingsToRetrieve + ), + this.retrieveSettings( + ENTITIES.INDEX_TEMPLATES, + settingsToRetrieve + ), + this.retrieveSettings( + ENTITIES.COMPONENT_TEMPLATES, + settingsToRetrieve + ), + ]); if (legacyTemplates) { this.legacyTemplate.loadTemplates(legacyTemplates); diff --git a/src/plugins/console/server/lib/proxy_request.ts b/src/plugins/console/server/lib/proxy_request.ts index 875345be52055..1cc6af9abc533 100644 --- a/src/plugins/console/server/lib/proxy_request.ts +++ b/src/plugins/console/server/lib/proxy_request.ts @@ -12,6 +12,7 @@ import net from 'net'; import stream from 'stream'; import Boom from '@hapi/boom'; import { URL } from 'url'; +import { trimStart } from 'lodash'; interface Args { method: 'get' | 'post' | 'put' | 'delete' | 'patch' | 'head'; @@ -23,6 +24,9 @@ interface Args { rejectUnauthorized?: boolean; } +const MAX_MAPPING_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB +const isMappingEndpoint = (pathname: string): boolean => trimStart(pathname, '/') === '_mapping'; + /** * Node http request library does not expect there to be trailing "[" or "]" * characters in ipv6 host names. @@ -76,9 +80,39 @@ export const proxyRequest = ({ agent, }); - req.once('response', (res) => { - resolved = true; - resolve(res); + req.on('response', (res) => { + // Check if the request is to _mapping endpoint and if so, limit the response to 10MB. This is to + // protect against large mapping responses that can cause the browser to hang. + if (isMappingEndpoint(pathname)) { + let responseSize = 0; + // Transform stream that limits the size of the response. If the response is larger than the + // MAX_MAPPING_RESPONSE_SIZE, the stream will emit an error. + const limitedResponse = new stream.Transform({ + transform(chunk, encoding, callback) { + responseSize += chunk.length; + if (responseSize > MAX_MAPPING_RESPONSE_SIZE) { + callback(Boom.badRequest('Maximum size of mappings response exceeded')); + } else { + callback(null, chunk); + } + }, + }); + + const source = res.pipe(limitedResponse); + source.on('error', (err) => { + reject(err); + }); + + source.on('finish', () => { + // we need to bind the pipe function to the new stream so that it can be used by consumers of the response stream + res.pipe = limitedResponse.pipe.bind(limitedResponse); + resolved = true; + resolve(res); + }); + } else { + resolved = true; + resolve(res); + } }); req.once('socket', (socket: net.Socket) => { From 809068515e67b20f38b8d4ef4915c3b9afdfb4ea Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Fri, 7 Oct 2022 13:48:14 +0500 Subject: [PATCH 06/17] Abort request if response exceedes the limit --- src/plugins/console/server/lib/proxy_request.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/console/server/lib/proxy_request.ts b/src/plugins/console/server/lib/proxy_request.ts index 1cc6af9abc533..e568dd7222864 100644 --- a/src/plugins/console/server/lib/proxy_request.ts +++ b/src/plugins/console/server/lib/proxy_request.ts @@ -80,7 +80,7 @@ export const proxyRequest = ({ agent, }); - req.on('response', (res) => { + req.once('response', (res) => { // Check if the request is to _mapping endpoint and if so, limit the response to 10MB. This is to // protect against large mapping responses that can cause the browser to hang. if (isMappingEndpoint(pathname)) { @@ -100,6 +100,7 @@ export const proxyRequest = ({ const source = res.pipe(limitedResponse); source.on('error', (err) => { + req.destroy(); reject(err); }); From 0288084d8e9ae789b09c726688713fa8ee076e27 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Mon, 10 Oct 2022 09:39:29 +0500 Subject: [PATCH 07/17] Address CR change --- .../console/public/services/autocomplete.ts | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index e4263bd2750ce..43e76898fe5f8 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -112,10 +112,10 @@ export class AutocompleteInfo { if (isMappingResponseExceededError) { this.notifications.toasts.addWarning({ - title: i18n.translate('devTools.autocomplete.mappingResponseExceededTitle', { + title: i18n.translate('console.autocomplete.mappingResponseExceededTitle', { defaultMessage: 'Maximum size of mappings response exceeded', }), - text: i18n.translate('devTools.autocomplete.mappingResponseExceededText', { + text: i18n.translate('console.autocomplete.mappingResponseExceededText', { defaultMessage: 'Some autocomplete suggestions may be missing. Disable autocomplete suggestions for fields in Settings to optimize performance.', }), @@ -196,32 +196,32 @@ export class AutocompleteInfo { } private async retrieveTemplates(settingsToRetrieve: SettingsToRetrieve) { - const [legacyTemplates, indexTemplates, componentTemplates] = await Promise.all([ + return Promise.allSettled([ this.retrieveSettings( ENTITIES.LEGACY_TEMPLATES, settingsToRetrieve - ), + ).then((legacyTemplates) => { + if (legacyTemplates) { + this.legacyTemplate.loadTemplates(legacyTemplates); + } + }), this.retrieveSettings( ENTITIES.INDEX_TEMPLATES, settingsToRetrieve - ), + ).then((indexTemplates) => { + if (indexTemplates && 'index_templates' in indexTemplates) { + this.indexTemplate.loadTemplates(indexTemplates); + } + }), this.retrieveSettings( ENTITIES.COMPONENT_TEMPLATES, settingsToRetrieve - ), + ).then((componentTemplates) => { + if (componentTemplates && 'component_templates' in componentTemplates) { + this.componentTemplate.loadTemplates(componentTemplates); + } + }), ]); - - if (legacyTemplates) { - this.legacyTemplate.loadTemplates(legacyTemplates); - } - - if (indexTemplates && 'index_templates' in indexTemplates) { - this.indexTemplate.loadTemplates(indexTemplates); - } - - if (componentTemplates && 'component_templates' in componentTemplates) { - this.componentTemplate.loadTemplates(componentTemplates); - } } public clearSubscriptions() { From f7114c1bd86387c1975781f94842ca3bb6954603 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Tue, 11 Oct 2022 12:29:50 +0500 Subject: [PATCH 08/17] Limit response size for autocomplete suggestions --- .../console/public/services/autocomplete.ts | 30 +++---- .../console/server/lib/proxy_request.ts | 47 +---------- src/plugins/console/server/lib/utils/index.ts | 1 + .../server/lib/utils/sanitize_hostname.ts | 14 ++++ .../console/autocomplete_entities/index.ts | 79 +++++++++++++++++++ src/plugins/console/server/routes/index.ts | 2 + 6 files changed, 114 insertions(+), 59 deletions(-) create mode 100644 src/plugins/console/server/lib/utils/sanitize_hostname.ts create mode 100644 src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 43e76898fe5f8..22b89d1b992de 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -19,7 +19,6 @@ import type { IndicesGetTemplateResponse, } from '@elastic/elasticsearch/lib/api/types'; import type { NotificationsSetup } from '@kbn/core-notifications-browser'; -import { send } from '../lib/es'; import { Alias, DataStream, @@ -103,21 +102,25 @@ export class AutocompleteInfo { ]).then((response) => { const errors = response.filter((result) => result.status === 'rejected'); if (errors.length) { - // Notify the user if mapping size is too large - const isMappingResponseExceededError = errors.some((error) => { + let path; + // Notify the user if response size is too large + const isResponseSizeTooLarge = errors.some((error) => { if ('reason' in error) { - return error.reason?.body?.message === 'Maximum size of mappings response exceeded'; + const url = new URL(error.reason.request?.url); + path = url.searchParams.get('path'); + + return error.reason.body?.message === 'Response size is too large'; } }); - if (isMappingResponseExceededError) { + if (isResponseSizeTooLarge) { this.notifications.toasts.addWarning({ - title: i18n.translate('console.autocomplete.mappingResponseExceededTitle', { - defaultMessage: 'Maximum size of mappings response exceeded', + title: i18n.translate('console.autocomplete.responseSizeTooLargeWarningTitle', { + defaultMessage: `Response size for {path} is too large`, + values: { path }, }), - text: i18n.translate('console.autocomplete.mappingResponseExceededText', { - defaultMessage: - 'Some autocomplete suggestions may be missing. Disable autocomplete suggestions for fields in Settings to optimize performance.', + text: i18n.translate('console.autocomplete.responseSizeTooLargeWarningText', { + defaultMessage: 'Some autocomplete suggestions may be missing.', }), }); } @@ -148,11 +151,8 @@ export class AutocompleteInfo { // Fetch autocomplete info if setting is enabled and if user has made changes. if (settingsToRetrieve[settingsKey]) { - return send({ - http: this.http, - method: 'GET', - // pretty=false to compress the response and save bandwidth by avoiding whitespace - path: `${settingKeyToPathMap[settingsKey]}?pretty=false`, + return this.http.get(`/api/console/autocomplete_entities`, { + query: { path: settingKeyToPathMap[settingsKey] }, asSystemRequest: true, }); } else { diff --git a/src/plugins/console/server/lib/proxy_request.ts b/src/plugins/console/server/lib/proxy_request.ts index e568dd7222864..90d3cafa8c566 100644 --- a/src/plugins/console/server/lib/proxy_request.ts +++ b/src/plugins/console/server/lib/proxy_request.ts @@ -12,7 +12,7 @@ import net from 'net'; import stream from 'stream'; import Boom from '@hapi/boom'; import { URL } from 'url'; -import { trimStart } from 'lodash'; +import { sanitizeHostname } from './utils'; interface Args { method: 'get' | 'post' | 'put' | 'delete' | 'patch' | 'head'; @@ -24,16 +24,6 @@ interface Args { rejectUnauthorized?: boolean; } -const MAX_MAPPING_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB -const isMappingEndpoint = (pathname: string): boolean => trimStart(pathname, '/') === '_mapping'; - -/** - * Node http request library does not expect there to be trailing "[" or "]" - * characters in ipv6 host names. - */ -const sanitizeHostname = (hostName: string): string => - hostName.trim().replace(/^\[/, '').replace(/\]$/, ''); - // We use a modified version of Hapi's Wreck because Hapi, Axios, and Superagent don't support GET requests // with bodies, but ES APIs do. Similarly with DELETE requests with bodies. Another library, `request` // diverged too much from current behaviour. @@ -81,39 +71,8 @@ export const proxyRequest = ({ }); req.once('response', (res) => { - // Check if the request is to _mapping endpoint and if so, limit the response to 10MB. This is to - // protect against large mapping responses that can cause the browser to hang. - if (isMappingEndpoint(pathname)) { - let responseSize = 0; - // Transform stream that limits the size of the response. If the response is larger than the - // MAX_MAPPING_RESPONSE_SIZE, the stream will emit an error. - const limitedResponse = new stream.Transform({ - transform(chunk, encoding, callback) { - responseSize += chunk.length; - if (responseSize > MAX_MAPPING_RESPONSE_SIZE) { - callback(Boom.badRequest('Maximum size of mappings response exceeded')); - } else { - callback(null, chunk); - } - }, - }); - - const source = res.pipe(limitedResponse); - source.on('error', (err) => { - req.destroy(); - reject(err); - }); - - source.on('finish', () => { - // we need to bind the pipe function to the new stream so that it can be used by consumers of the response stream - res.pipe = limitedResponse.pipe.bind(limitedResponse); - resolved = true; - resolve(res); - }); - } else { - resolved = true; - resolve(res); - } + resolved = true; + resolve(res); }); req.once('socket', (socket: net.Socket) => { diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index 2c595640eefbf..5bc86de56c11e 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -9,3 +9,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; export { streamToJSON } from './stream_to_json'; +export { sanitizeHostname } from './sanitize_hostname'; diff --git a/src/plugins/console/server/lib/utils/sanitize_hostname.ts b/src/plugins/console/server/lib/utils/sanitize_hostname.ts new file mode 100644 index 0000000000000..a7b49c5874f58 --- /dev/null +++ b/src/plugins/console/server/lib/utils/sanitize_hostname.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Node http request library does not expect there to be trailing "[" or "]" + * characters in ipv6 host names. + */ +export const sanitizeHostname = (hostName: string): string => + hostName.trim().replace(/^\[/, '').replace(/\]$/, ''); diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts new file mode 100644 index 0000000000000..c581a6fefbec0 --- /dev/null +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import http from 'http'; +import https from 'https'; +import Boom from '@hapi/boom'; +import { RouteDependencies } from '../../..'; +import { sanitizeHostname } from '../../../../lib/utils'; + +export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { + deps.router.get( + { + path: '/api/console/autocomplete_entities', + options: { + tags: ['access:console'], + }, + validate: false, + }, + async (context, request, response) => { + const legacyConfig = await deps.proxy.readLegacyESConfig(); + const { hosts } = legacyConfig; + const path = request.url.searchParams.get('path'); + let body; + + for (let idx = 0; idx < hosts.length; idx++) { + const host = hosts[idx]; + const { hostname, port, protocol } = new URL(host); + const client = protocol === 'https:' ? https : http; + const options = { + method: 'GET', + host: sanitizeHostname(hostname), + port: port === '' ? undefined : parseInt(port, 10), + protocol, + path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace + }; + + try { + body = await new Promise((resolve, reject) => { + const req = client.request(options, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + + // Limit the size of the response to 10MB + if (Buffer.byteLength(Buffer.concat(chunks)) > 10 * 1024 * 1024) { + req.destroy(); + reject(Boom.badRequest('Response size is too large')); + } + }); + res.on('end', () => { + resolve(Buffer.concat(chunks).toString('utf8')); + }); + }); + req.on('error', reject); + req.end(); + }); + break; + } catch (err) { + if (idx === hosts.length - 1) { + return response.customError({ + statusCode: 500, + body: err, + }); + } + // Try the next host + } + } + + return response.ok({ + body, + }); + } + ); +}; diff --git a/src/plugins/console/server/routes/index.ts b/src/plugins/console/server/routes/index.ts index 8554c6461818f..e1a036f55a62c 100644 --- a/src/plugins/console/server/routes/index.ts +++ b/src/plugins/console/server/routes/index.ts @@ -17,6 +17,7 @@ import { handleEsError } from '../shared_imports'; import { registerEsConfigRoute } from './api/console/es_config'; import { registerProxyRoute } from './api/console/proxy'; import { registerSpecDefinitionsRoute } from './api/console/spec_definitions'; +import { registerAutocompleteEntitiesRoute } from './api/console/autocomplete_entities'; export interface ProxyDependencies { readLegacyESConfig: () => Promise; @@ -42,4 +43,5 @@ export const registerRoutes = (dependencies: RouteDependencies) => { registerEsConfigRoute(dependencies); registerProxyRoute(dependencies); registerSpecDefinitionsRoute(dependencies); + registerAutocompleteEntitiesRoute(dependencies); }; From 54b461692b86cf4d0688af7672482d76865072d4 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 10:12:32 +0500 Subject: [PATCH 09/17] Handle fetching entities on serverside --- .../application/containers/settings.tsx | 5 +- .../public/lib/autocomplete_entities/types.ts | 18 ++ src/plugins/console/public/plugin.ts | 2 +- .../console/public/services/autocomplete.ts | 200 ++++------------- .../console/autocomplete_entities/index.ts | 203 +++++++++++++----- 5 files changed, 212 insertions(+), 216 deletions(-) diff --git a/src/plugins/console/public/application/containers/settings.tsx b/src/plugins/console/public/application/containers/settings.tsx index 878f6dc64eb74..b9a9d68294e6d 100644 --- a/src/plugins/console/public/application/containers/settings.tsx +++ b/src/plugins/console/public/application/containers/settings.tsx @@ -13,7 +13,6 @@ import { AutocompleteOptions, DevToolsSettingsModal } from '../components'; import { useServicesContext, useEditorActionContext } from '../contexts'; import { DevToolsSettings, Settings as SettingsService } from '../../services'; import type { SenseEditor } from '../models'; -import type { SettingsToRetrieve } from '../../services'; const getAutocompleteDiff = ( newSettings: DevToolsSettings, @@ -39,7 +38,7 @@ export function Settings({ onClose, editorInstance }: Props) { const refreshAutocompleteSettings = ( settingsService: SettingsService, - selectedSettings: SettingsToRetrieve + selectedSettings: DevToolsSettings['autocomplete'] ) => { autocompleteInfo.retrieve(settingsService, selectedSettings); }; @@ -100,7 +99,7 @@ export function Settings({ onClose, editorInstance }: Props) { onClose={onClose} onSaveSettings={onSaveSettings} refreshAutocompleteSettings={(selectedSettings) => - refreshAutocompleteSettings(settings, selectedSettings as SettingsToRetrieve) + refreshAutocompleteSettings(settings, selectedSettings) } settings={settings.toJSON()} editorInstance={editorInstance} diff --git a/src/plugins/console/public/lib/autocomplete_entities/types.ts b/src/plugins/console/public/lib/autocomplete_entities/types.ts index 38a280193b61c..c3aa06d541be6 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/types.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/types.ts @@ -6,6 +6,15 @@ * Side Public License, v 1. */ +import type { + ClusterGetComponentTemplateResponse, + IndicesGetAliasResponse, + IndicesGetDataStreamResponse, + IndicesGetIndexTemplateResponse, + IndicesGetMappingResponse, + IndicesGetTemplateResponse, +} from '@elastic/elasticsearch/lib/api/types'; + export interface Field { name: string; type: string; @@ -19,3 +28,12 @@ export interface FieldMapping { index_name?: string; fields?: FieldMapping[]; } + +export interface MappingsApiResponse { + mappings: IndicesGetMappingResponse; + aliases: IndicesGetAliasResponse; + dataStreams: IndicesGetDataStreamResponse; + legacyTemplates: IndicesGetTemplateResponse; + indexTemplates: IndicesGetIndexTemplateResponse; + componentTemplates: ClusterGetComponentTemplateResponse; +} \ No newline at end of file diff --git a/src/plugins/console/public/plugin.ts b/src/plugins/console/public/plugin.ts index 886759e2dc896..33ee5446dc268 100644 --- a/src/plugins/console/public/plugin.ts +++ b/src/plugins/console/public/plugin.ts @@ -29,7 +29,7 @@ export class ConsoleUIPlugin implements Plugin(); - this.autocompleteInfo.setup(http, notifications); + this.autocompleteInfo.setup(http); setAutocompleteInfo(this.autocompleteInfo); if (isConsoleUiEnabled) { diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 22b89d1b992de..729eb9e08d9ee 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -7,18 +7,9 @@ */ import { createGetterSetter } from '@kbn/kibana-utils-plugin/public'; -import { i18n } from '@kbn/i18n'; - import type { HttpSetup } from '@kbn/core/public'; -import type { - ClusterGetComponentTemplateResponse, - IndicesGetAliasResponse, - IndicesGetDataStreamResponse, - IndicesGetIndexTemplateResponse, - IndicesGetMappingResponse, - IndicesGetTemplateResponse, -} from '@elastic/elasticsearch/lib/api/types'; -import type { NotificationsSetup } from '@kbn/core-notifications-browser'; +import type { MappingsApiResponse } from '../lib/autocomplete_entities/types'; +import { API_BASE_PATH } from '../../common/constants'; import { Alias, DataStream, @@ -27,7 +18,7 @@ import { IndexTemplate, ComponentTemplate, } from '../lib/autocomplete_entities'; -import type { DevToolsSettings, Settings } from './settings'; +import { DevToolsSettings, Settings } from './settings'; export enum ENTITIES { INDICES = 'indices', @@ -38,13 +29,6 @@ export enum ENTITIES { DATA_STREAMS = 'dataStreams', } -export type SettingsToRetrieve = DevToolsSettings['autocomplete'] & { - indexTemplates: boolean; - componentTemplates: boolean; - legacyTemplates: boolean; -}; -type SettingsKey = keyof Omit; - export class AutocompleteInfo { public readonly alias = new Alias(); public readonly mapping = new Mapping(); @@ -53,12 +37,10 @@ export class AutocompleteInfo { public readonly indexTemplate = new IndexTemplate(); public readonly componentTemplate = new ComponentTemplate(); private http!: HttpSetup; - private notifications!: NotificationsSetup; private pollTimeoutId: ReturnType | undefined; - public setup(http: HttpSetup, notifications: NotificationsSetup) { + public setup(http: HttpSetup) { this.http = http; - this.notifications = notifications; } public getEntityProvider( @@ -66,162 +48,42 @@ export class AutocompleteInfo { context: { indices: string[]; types: string[] } = { indices: [], types: [] } ) { switch (type) { - case ENTITIES.INDICES: + case 'indices': const includeAliases = true; const collaborator = this.mapping; return () => this.alias.getIndices(includeAliases, collaborator); - case ENTITIES.FIELDS: + case 'fields': return this.mapping.getMappings(context.indices, context.types); - case ENTITIES.INDEX_TEMPLATES: + case 'indexTemplates': return () => this.indexTemplate.getTemplates(); - case ENTITIES.COMPONENT_TEMPLATES: + case 'componentTemplates': return () => this.componentTemplate.getTemplates(); - case ENTITIES.LEGACY_TEMPLATES: + case 'legacyTemplates': return () => this.legacyTemplate.getTemplates(); - case ENTITIES.DATA_STREAMS: + case 'dataStreams': return () => this.dataStream.getDataStreams(); default: - throw new Error(`Unknown entity type: ${type}`); + throw new Error(`Unsupported type: ${type}`); } } - public retrieve(settings: Settings, settingsToRetrieve: SettingsToRetrieve) { + public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) { this.clearSubscriptions(); - const templateSettingsToRetrieve = { - ...settingsToRetrieve, - legacyTemplates: settingsToRetrieve.templates, - indexTemplates: settingsToRetrieve.templates, - componentTemplates: settingsToRetrieve.templates, - }; - - Promise.allSettled([ - this.retrieveMappings(settingsToRetrieve), - this.retrieveAliases(settingsToRetrieve), - this.retrieveDataStreams(settingsToRetrieve), - this.retrieveTemplates(templateSettingsToRetrieve), - ]).then((response) => { - const errors = response.filter((result) => result.status === 'rejected'); - if (errors.length) { - let path; - // Notify the user if response size is too large - const isResponseSizeTooLarge = errors.some((error) => { - if ('reason' in error) { - const url = new URL(error.reason.request?.url); - path = url.searchParams.get('path'); - - return error.reason.body?.message === 'Response size is too large'; + this.http + .get(`${API_BASE_PATH}/autocomplete_entities`, { + query: { ...settingsToRetrieve }, + }) + .then((data) => { + this.load(data); + // Schedule next request. + this.pollTimeoutId = setTimeout(() => { + // This looks strange/inefficient, but it ensures correct behavior because we don't want to send + // a scheduled request if the user turns off polling. + if (settings.getPolling()) { + this.retrieve(settings, settings.getAutocomplete()); } - }); - - if (isResponseSizeTooLarge) { - this.notifications.toasts.addWarning({ - title: i18n.translate('console.autocomplete.responseSizeTooLargeWarningTitle', { - defaultMessage: `Response size for {path} is too large`, - values: { path }, - }), - text: i18n.translate('console.autocomplete.responseSizeTooLargeWarningText', { - defaultMessage: 'Some autocomplete suggestions may be missing.', - }), - }); - } - } - - this.pollTimeoutId = setTimeout(() => { - // This looks strange/inefficient, but it ensures correct behavior because we don't want to send - // a scheduled request if the user turns off polling. - if (settings.getPolling()) { - this.retrieve(settings, settings.getAutocomplete()); - } - }, settings.getPollInterval()); - }); - } - - private retrieveSettings( - settingsKey: SettingsKey, - settingsToRetrieve: SettingsToRetrieve - ): Promise { - const settingKeyToPathMap = { - fields: '_mapping', - indices: '_aliases', - legacyTemplates: '_template', - indexTemplates: '_index_template', - componentTemplates: '_component_template', - dataStreams: '_data_stream', - }; - - // Fetch autocomplete info if setting is enabled and if user has made changes. - if (settingsToRetrieve[settingsKey]) { - return this.http.get(`/api/console/autocomplete_entities`, { - query: { path: settingKeyToPathMap[settingsKey] }, - asSystemRequest: true, + }, settings.getPollInterval()); }); - } else { - // If the user doesn't want autocomplete suggestions, then clear any that exist - return Promise.resolve({}); - } - } - - private async retrieveMappings(settingsToRetrieve: SettingsToRetrieve) { - const mappings = await this.retrieveSettings( - ENTITIES.FIELDS, - settingsToRetrieve - ); - - if (mappings) { - this.mapping.loadMappings(mappings); - } - } - - private async retrieveAliases(settingsToRetrieve: SettingsToRetrieve) { - const aliases = await this.retrieveSettings( - ENTITIES.INDICES, - settingsToRetrieve - ); - - if (aliases) { - const collaborator = this.mapping; - this.alias.loadAliases(aliases, collaborator); - } - } - - private async retrieveDataStreams(settingsToRetrieve: SettingsToRetrieve) { - const dataStreams = await this.retrieveSettings( - ENTITIES.DATA_STREAMS, - settingsToRetrieve - ); - - if (dataStreams && 'data_streams' in dataStreams) { - this.dataStream.loadDataStreams(dataStreams); - } - } - - private async retrieveTemplates(settingsToRetrieve: SettingsToRetrieve) { - return Promise.allSettled([ - this.retrieveSettings( - ENTITIES.LEGACY_TEMPLATES, - settingsToRetrieve - ).then((legacyTemplates) => { - if (legacyTemplates) { - this.legacyTemplate.loadTemplates(legacyTemplates); - } - }), - this.retrieveSettings( - ENTITIES.INDEX_TEMPLATES, - settingsToRetrieve - ).then((indexTemplates) => { - if (indexTemplates && 'index_templates' in indexTemplates) { - this.indexTemplate.loadTemplates(indexTemplates); - } - }), - this.retrieveSettings( - ENTITIES.COMPONENT_TEMPLATES, - settingsToRetrieve - ).then((componentTemplates) => { - if (componentTemplates && 'component_templates' in componentTemplates) { - this.componentTemplate.loadTemplates(componentTemplates); - } - }), - ]); } public clearSubscriptions() { @@ -230,6 +92,16 @@ export class AutocompleteInfo { } } + private load(data: MappingsApiResponse) { + this.mapping.loadMappings(data.mappings); + const collaborator = this.mapping; + this.alias.loadAliases(data.aliases, collaborator); + this.indexTemplate.loadTemplates(data.indexTemplates); + this.componentTemplate.loadTemplates(data.componentTemplates); + this.legacyTemplate.loadTemplates(data.legacyTemplates); + this.dataStream.loadDataStreams(data.dataStreams); + } + public clear() { this.alias.clearAliases(); this.mapping.clearMappings(); @@ -241,4 +113,4 @@ export class AutocompleteInfo { } export const [getAutocompleteInfo, setAutocompleteInfo] = - createGetterSetter('AutocompleteInfo'); + createGetterSetter('AutocompleteInfo'); \ No newline at end of file diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index c581a6fefbec0..b1bab856acb31 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -8,9 +8,73 @@ import http from 'http'; import https from 'https'; +import { parse } from 'query-string'; import Boom from '@hapi/boom'; -import { RouteDependencies } from '../../..'; +import type { RouteDependencies } from '../../..'; import { sanitizeHostname } from '../../../../lib/utils'; +import type { ESConfigForProxy } from '@kbn/console-plugin/server/types'; + +interface Settings { + indices: boolean; + fields: boolean; + templates: boolean; + dataStreams: boolean; +} + +const getMappings = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.fields) { + const mappings = await getEntity(config, '/_mapping'); + return mappings; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +const getAliases = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.indices) { + const aliases = await getEntity(config, '/_alias'); + return aliases; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +const getDataStreams = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.dataStreams) { + const dataStreams = await getEntity(config, '/_data_stream'); + return dataStreams; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +const getLegacyTemplates = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.templates) { + const legacyTemplates = await getEntity(config, '/_template'); + return legacyTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +const getIndexTemplates = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.templates) { + const indexTemplates = await getEntity(config, '/_index_template'); + return indexTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +const getComponentTemplates = async (settings: Settings, config: ESConfigForProxy) => { + if (settings.templates) { + const componentTemplates = await getEntity(config, '/_component_template'); + return componentTemplates; + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { deps.router.get( @@ -22,58 +86,101 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { validate: false, }, async (context, request, response) => { + const settings = parse(request.url.search, { parseBooleans: true }) as unknown as Settings; + + // If no settings are provided return 400 + if (Object.keys(settings).length === 0) { + return response.badRequest({ + body: 'Request must contain a query param of autocomplete settings', + }); + } + + + const legacyConfig = await deps.proxy.readLegacyESConfig(); - const { hosts } = legacyConfig; - const path = request.url.searchParams.get('path'); - let body; - - for (let idx = 0; idx < hosts.length; idx++) { - const host = hosts[idx]; - const { hostname, port, protocol } = new URL(host); - const client = protocol === 'https:' ? https : http; - const options = { - method: 'GET', - host: sanitizeHostname(hostname), - port: port === '' ? undefined : parseInt(port, 10), - protocol, - path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace - }; - - try { - body = await new Promise((resolve, reject) => { - const req = client.request(options, (res) => { - const chunks: Buffer[] = []; - res.on('data', (chunk) => { - chunks.push(chunk); - - // Limit the size of the response to 10MB - if (Buffer.byteLength(Buffer.concat(chunks)) > 10 * 1024 * 1024) { - req.destroy(); - reject(Boom.badRequest('Response size is too large')); - } - }); - res.on('end', () => { - resolve(Buffer.concat(chunks).toString('utf8')); - }); - }); - req.on('error', reject); - req.end(); - }); - break; - } catch (err) { - if (idx === hosts.length - 1) { - return response.customError({ - statusCode: 500, - body: err, - }); - } - // Try the next host + + // Wait for all requests to complete, in case one of them fails return the successfull ones + const results = await Promise.allSettled([ + getMappings(settings, legacyConfig), + getAliases(settings, legacyConfig), + getDataStreams(settings, legacyConfig), + getLegacyTemplates(settings, legacyConfig), + getIndexTemplates(settings, legacyConfig), + getComponentTemplates(settings, legacyConfig), + ]); + + const [ + mappings, + aliases, + dataStreams, + legacyTemplates, + indexTemplates, + componentTemplates, + ] = results.map((result) => { + // If the request was successful, return the result + if (result.status === 'fulfilled') { + return result.value; } - } + // If the request failed, return an empty object + return {}; + }); return response.ok({ - body, + body: { + mappings, + aliases, + dataStreams, + legacyTemplates, + indexTemplates, + componentTemplates, + }, }); } ); }; + + +const getEntity = (legacyConfig: ESConfigForProxy, path: string) => { + return new Promise((resolve, reject) => { + const { hosts } = legacyConfig; + for (let idx = 0; idx < hosts.length; idx++) { + const host = hosts[idx]; + const { hostname, port, protocol } = new URL(host); + const client = protocol === 'https:' ? https : http; + const options = { + method: 'GET', + host: sanitizeHostname(hostname), + port: port === '' ? undefined : parseInt(port, 10), + protocol, + path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace + }; + + try { + const req = client.request(options, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + + // Limit the size of the response to 10MB + if (Buffer.byteLength(Buffer.concat(chunks)) > 10 * 1024 * 1024) { + req.destroy(); + reject(Boom.badRequest('Response size is too large')); + } + }); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + resolve(JSON.parse(body)); + }); + }); + req.on('error', reject); + req.end(); + break; + } catch (err) { + if (idx === hosts.length - 1) { + reject(err); + } + // Try the next host + } + } + }); +} \ No newline at end of file From a90fee8f6f20927a7828a09c5fa16804371e0075 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 11:44:43 +0500 Subject: [PATCH 10/17] Update and resolve conflicts --- .../public/lib/autocomplete_entities/types.ts | 4 +- .../console/public/services/autocomplete.ts | 20 +-- src/plugins/console/server/lib/utils/index.ts | 1 - .../server/lib/utils/stream_to_json.test.ts | 35 ---- .../server/lib/utils/stream_to_json.ts | 27 ---- .../console/autocomplete_entities/index.ts | 149 +++++++++--------- 6 files changed, 84 insertions(+), 152 deletions(-) delete mode 100644 src/plugins/console/server/lib/utils/stream_to_json.test.ts delete mode 100644 src/plugins/console/server/lib/utils/stream_to_json.ts diff --git a/src/plugins/console/public/lib/autocomplete_entities/types.ts b/src/plugins/console/public/lib/autocomplete_entities/types.ts index c3aa06d541be6..9c2607b8ad84f 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/types.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/types.ts @@ -29,11 +29,11 @@ export interface FieldMapping { fields?: FieldMapping[]; } -export interface MappingsApiResponse { +export interface AutoCompleteEntitiesApiResponse { mappings: IndicesGetMappingResponse; aliases: IndicesGetAliasResponse; dataStreams: IndicesGetDataStreamResponse; legacyTemplates: IndicesGetTemplateResponse; indexTemplates: IndicesGetIndexTemplateResponse; componentTemplates: ClusterGetComponentTemplateResponse; -} \ No newline at end of file +} diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 729eb9e08d9ee..3e1a38a514607 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -8,7 +8,7 @@ import { createGetterSetter } from '@kbn/kibana-utils-plugin/public'; import type { HttpSetup } from '@kbn/core/public'; -import type { MappingsApiResponse } from '../lib/autocomplete_entities/types'; +import type { AutoCompleteEntitiesApiResponse } from '../lib/autocomplete_entities/types'; import { API_BASE_PATH } from '../../common/constants'; import { Alias, @@ -48,19 +48,19 @@ export class AutocompleteInfo { context: { indices: string[]; types: string[] } = { indices: [], types: [] } ) { switch (type) { - case 'indices': + case ENTITIES.INDICES: const includeAliases = true; const collaborator = this.mapping; return () => this.alias.getIndices(includeAliases, collaborator); - case 'fields': + case ENTITIES.FIELDS: return this.mapping.getMappings(context.indices, context.types); - case 'indexTemplates': + case ENTITIES.INDEX_TEMPLATES: return () => this.indexTemplate.getTemplates(); - case 'componentTemplates': + case ENTITIES.COMPONENT_TEMPLATES: return () => this.componentTemplate.getTemplates(); - case 'legacyTemplates': + case ENTITIES.LEGACY_TEMPLATES: return () => this.legacyTemplate.getTemplates(); - case 'dataStreams': + case ENTITIES.DATA_STREAMS: return () => this.dataStream.getDataStreams(); default: throw new Error(`Unsupported type: ${type}`); @@ -70,7 +70,7 @@ export class AutocompleteInfo { public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) { this.clearSubscriptions(); this.http - .get(`${API_BASE_PATH}/autocomplete_entities`, { + .get(`${API_BASE_PATH}/autocomplete_entities`, { query: { ...settingsToRetrieve }, }) .then((data) => { @@ -92,7 +92,7 @@ export class AutocompleteInfo { } } - private load(data: MappingsApiResponse) { + private load(data: AutoCompleteEntitiesApiResponse) { this.mapping.loadMappings(data.mappings); const collaborator = this.mapping; this.alias.loadAliases(data.aliases, collaborator); @@ -113,4 +113,4 @@ export class AutocompleteInfo { } export const [getAutocompleteInfo, setAutocompleteInfo] = - createGetterSetter('AutocompleteInfo'); \ No newline at end of file + createGetterSetter('AutocompleteInfo'); diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index 5bc86de56c11e..aac0f890a8a80 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -8,5 +8,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; -export { streamToJSON } from './stream_to_json'; export { sanitizeHostname } from './sanitize_hostname'; diff --git a/src/plugins/console/server/lib/utils/stream_to_json.test.ts b/src/plugins/console/server/lib/utils/stream_to_json.test.ts deleted file mode 100644 index 780d3adb4a145..0000000000000 --- a/src/plugins/console/server/lib/utils/stream_to_json.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { Readable } from 'stream'; -import { streamToJSON } from './stream_to_json'; -import type { IncomingMessage } from 'http'; - -describe('streamToString', () => { - it('should limit the response size', async () => { - const stream = new Readable({ - read() { - this.push('a'.repeat(1000)); - }, - }); - await expect( - streamToJSON(stream as IncomingMessage, 500) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`); - }); - - it('should parse the response', async () => { - const stream = new Readable({ - read() { - this.push('{"test": "test"}'); - this.push(null); - }, - }); - const result = await streamToJSON(stream as IncomingMessage, 5000); - expect(result).toEqual({ test: 'test' }); - }); -}); diff --git a/src/plugins/console/server/lib/utils/stream_to_json.ts b/src/plugins/console/server/lib/utils/stream_to_json.ts deleted file mode 100644 index 5ff1974e9fd47..0000000000000 --- a/src/plugins/console/server/lib/utils/stream_to_json.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { IncomingMessage } from 'http'; - -export function streamToJSON(stream: IncomingMessage, limit: number) { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on('data', (chunk) => { - chunks.push(chunk); - if (Buffer.byteLength(Buffer.concat(chunks)) > limit) { - stream.destroy(); - reject(new Error('Response size limit exceeded')); - } - }); - stream.on('end', () => { - const response = Buffer.concat(chunks).toString('utf8'); - resolve(JSON.parse(response)); - }); - stream.on('error', reject); - }); -} diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index b1bab856acb31..e0fac91120a16 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -8,11 +8,12 @@ import http from 'http'; import https from 'https'; +import { Buffer } from 'buffer'; import { parse } from 'query-string'; import Boom from '@hapi/boom'; import type { RouteDependencies } from '../../..'; import { sanitizeHostname } from '../../../../lib/utils'; -import type { ESConfigForProxy } from '@kbn/console-plugin/server/types'; +import type { ESConfigForProxy } from '../../../../types'; interface Settings { indices: boolean; @@ -21,60 +22,108 @@ interface Settings { dataStreams: boolean; } +const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB +// Limit the response size to 10MB, because the response can be very large and sending it to the client +// can cause the browser to hang. + const getMappings = async (settings: Settings, config: ESConfigForProxy) => { if (settings.fields) { - const mappings = await getEntity(config, '/_mapping'); + const mappings = await getEntity('/_mapping', config); return mappings; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; const getAliases = async (settings: Settings, config: ESConfigForProxy) => { if (settings.indices) { - const aliases = await getEntity(config, '/_alias'); + const aliases = await getEntity('/_alias', config); return aliases; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; const getDataStreams = async (settings: Settings, config: ESConfigForProxy) => { if (settings.dataStreams) { - const dataStreams = await getEntity(config, '/_data_stream'); + const dataStreams = await getEntity('/_data_stream', config); return dataStreams; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; const getLegacyTemplates = async (settings: Settings, config: ESConfigForProxy) => { if (settings.templates) { - const legacyTemplates = await getEntity(config, '/_template'); + const legacyTemplates = await getEntity('/_template', config); return legacyTemplates; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; const getIndexTemplates = async (settings: Settings, config: ESConfigForProxy) => { if (settings.templates) { - const indexTemplates = await getEntity(config, '/_index_template'); + const indexTemplates = await getEntity('/_index_template', config); return indexTemplates; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; const getComponentTemplates = async (settings: Settings, config: ESConfigForProxy) => { if (settings.templates) { - const componentTemplates = await getEntity(config, '/_component_template'); + const componentTemplates = await getEntity('/_component_template', config); return componentTemplates; } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; -} +}; + +const getEntity = (path: string, config: ESConfigForProxy) => { + return new Promise((resolve, reject) => { + const { hosts } = config; + for (let idx = 0; idx < hosts.length; idx++) { + const host = hosts[idx]; + const { hostname, port, protocol } = new URL(host); + const client = protocol === 'https:' ? https : http; + const options = { + method: 'GET', + host: sanitizeHostname(hostname), + port: port === '' ? undefined : parseInt(port, 10), + protocol, + path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace + }; + try { + const req = client.request(options, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + + // Destroy the request if the response is too large + if (Buffer.byteLength(Buffer.concat(chunks)) > MAX_RESPONSE_SIZE) { + req.destroy(); + reject(Boom.badRequest('Response size is too large')); + } + }); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + resolve(JSON.parse(body)); + }); + }); + req.on('error', reject); + req.end(); + break; + } catch (err) { + if (idx === hosts.length - 1) { + reject(err); + } + // Try the next host + } + } + }); +}; export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { deps.router.get( @@ -95,10 +144,8 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { }); } - - const legacyConfig = await deps.proxy.readLegacyESConfig(); - + // Wait for all requests to complete, in case one of them fails return the successfull ones const results = await Promise.allSettled([ getMappings(settings, legacyConfig), @@ -109,21 +156,15 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { getComponentTemplates(settings, legacyConfig), ]); - const [ - mappings, - aliases, - dataStreams, - legacyTemplates, - indexTemplates, - componentTemplates, - ] = results.map((result) => { - // If the request was successful, return the result - if (result.status === 'fulfilled') { - return result.value; - } - // If the request failed, return an empty object - return {}; - }); + const [mappings, aliases, dataStreams, legacyTemplates, indexTemplates, componentTemplates] = + results.map((result) => { + // If the request was successful, return the result + if (result.status === 'fulfilled') { + return result.value; + } + // If the request failed, return an empty object + return {}; + }); return response.ok({ body: { @@ -138,49 +179,3 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { } ); }; - - -const getEntity = (legacyConfig: ESConfigForProxy, path: string) => { - return new Promise((resolve, reject) => { - const { hosts } = legacyConfig; - for (let idx = 0; idx < hosts.length; idx++) { - const host = hosts[idx]; - const { hostname, port, protocol } = new URL(host); - const client = protocol === 'https:' ? https : http; - const options = { - method: 'GET', - host: sanitizeHostname(hostname), - port: port === '' ? undefined : parseInt(port, 10), - protocol, - path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace - }; - - try { - const req = client.request(options, (res) => { - const chunks: Buffer[] = []; - res.on('data', (chunk) => { - chunks.push(chunk); - - // Limit the size of the response to 10MB - if (Buffer.byteLength(Buffer.concat(chunks)) > 10 * 1024 * 1024) { - req.destroy(); - reject(Boom.badRequest('Response size is too large')); - } - }); - res.on('end', () => { - const body = Buffer.concat(chunks).toString('utf8'); - resolve(JSON.parse(body)); - }); - }); - req.on('error', reject); - req.end(); - break; - } catch (err) { - if (idx === hosts.length - 1) { - reject(err); - } - // Try the next host - } - } - }); -} \ No newline at end of file From 4afa68589983a25d8273128065f1e5a0868a25d5 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 12:34:13 +0500 Subject: [PATCH 11/17] Fix type checks --- src/plugins/console/public/services/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/console/public/services/index.ts b/src/plugins/console/public/services/index.ts index 01d190d4e150e..73929f89e386f 100644 --- a/src/plugins/console/public/services/index.ts +++ b/src/plugins/console/public/services/index.ts @@ -16,4 +16,3 @@ export { setAutocompleteInfo, ENTITIES, } from './autocomplete'; -export type { SettingsToRetrieve } from './autocomplete'; From c2121f95983ec5ee6f7f3d0a657ef2d1e50775e0 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 13:27:00 +0500 Subject: [PATCH 12/17] Log the error which caused the request to fail --- .../routes/api/console/autocomplete_entities/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index e0fac91120a16..220168e5e0af2 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -104,7 +104,7 @@ const getEntity = (path: string, config: ESConfigForProxy) => { // Destroy the request if the response is too large if (Buffer.byteLength(Buffer.concat(chunks)) > MAX_RESPONSE_SIZE) { req.destroy(); - reject(Boom.badRequest('Response size is too large')); + reject(Boom.badRequest(`Response size is too large for ${path}`)); } }); res.on('end', () => { @@ -162,7 +162,12 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { if (result.status === 'fulfilled') { return result.value; } - // If the request failed, return an empty object + + // If the request failed, log the error and return an empty object + if (result.reason instanceof Error) { + deps.log.error(result.reason); + } + return {}; }); From d5c2fddccbf6204c951634c062e961d5d9a50eb5 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 17:11:02 +0500 Subject: [PATCH 13/17] Fix request headers and restore integration tests --- .../console/autocomplete_entities/index.ts | 43 ++++++++++++------- .../api/console/proxy/create_handler.ts | 2 +- test/api_integration/apis/console/index.ts | 1 + 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index 220168e5e0af2..49fbdb9a3659a 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -11,9 +11,12 @@ import https from 'https'; import { Buffer } from 'buffer'; import { parse } from 'query-string'; import Boom from '@hapi/boom'; +import type { KibanaRequest } from '@kbn/core-http-server'; +import type { SemVer } from 'semver'; import type { RouteDependencies } from '../../..'; import { sanitizeHostname } from '../../../../lib/utils'; import type { ESConfigForProxy } from '../../../../types'; +import { getRequestConfig } from '../proxy/create_handler'; interface Settings { indices: boolean; @@ -22,11 +25,13 @@ interface Settings { dataStreams: boolean; } +type Config = ESConfigForProxy & { headers: KibanaRequest['headers'] } & { kibanaVersion: SemVer }; + const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB // Limit the response size to 10MB, because the response can be very large and sending it to the client // can cause the browser to hang. -const getMappings = async (settings: Settings, config: ESConfigForProxy) => { +const getMappings = async (settings: Settings, config: Config) => { if (settings.fields) { const mappings = await getEntity('/_mapping', config); return mappings; @@ -35,7 +40,7 @@ const getMappings = async (settings: Settings, config: ESConfigForProxy) => { return {}; }; -const getAliases = async (settings: Settings, config: ESConfigForProxy) => { +const getAliases = async (settings: Settings, config: Config) => { if (settings.indices) { const aliases = await getEntity('/_alias', config); return aliases; @@ -44,7 +49,7 @@ const getAliases = async (settings: Settings, config: ESConfigForProxy) => { return {}; }; -const getDataStreams = async (settings: Settings, config: ESConfigForProxy) => { +const getDataStreams = async (settings: Settings, config: Config) => { if (settings.dataStreams) { const dataStreams = await getEntity('/_data_stream', config); return dataStreams; @@ -53,7 +58,7 @@ const getDataStreams = async (settings: Settings, config: ESConfigForProxy) => { return {}; }; -const getLegacyTemplates = async (settings: Settings, config: ESConfigForProxy) => { +const getLegacyTemplates = async (settings: Settings, config: Config) => { if (settings.templates) { const legacyTemplates = await getEntity('/_template', config); return legacyTemplates; @@ -62,7 +67,7 @@ const getLegacyTemplates = async (settings: Settings, config: ESConfigForProxy) return {}; }; -const getIndexTemplates = async (settings: Settings, config: ESConfigForProxy) => { +const getIndexTemplates = async (settings: Settings, config: Config) => { if (settings.templates) { const indexTemplates = await getEntity('/_index_template', config); return indexTemplates; @@ -71,7 +76,7 @@ const getIndexTemplates = async (settings: Settings, config: ESConfigForProxy) = return {}; }; -const getComponentTemplates = async (settings: Settings, config: ESConfigForProxy) => { +const getComponentTemplates = async (settings: Settings, config: Config) => { if (settings.templates) { const componentTemplates = await getEntity('/_component_template', config); return componentTemplates; @@ -80,15 +85,18 @@ const getComponentTemplates = async (settings: Settings, config: ESConfigForProx return {}; }; -const getEntity = (path: string, config: ESConfigForProxy) => { +const getEntity = (path: string, config: Config) => { return new Promise((resolve, reject) => { - const { hosts } = config; + const { hosts, kibanaVersion } = config; for (let idx = 0; idx < hosts.length; idx++) { const host = hosts[idx]; - const { hostname, port, protocol } = new URL(host); + const uri = new URL(host + path); + const { protocol, hostname, port } = uri; + const { headers } = getRequestConfig(config.headers, config, uri.toString(), kibanaVersion); const client = protocol === 'https:' ? https : http; const options = { method: 'GET', + headers: { ...headers }, host: sanitizeHostname(hostname), port: port === '' ? undefined : parseInt(port, 10), protocol, @@ -145,15 +153,20 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { } const legacyConfig = await deps.proxy.readLegacyESConfig(); + const configWithHeaders = { + ...legacyConfig, + headers: request.headers, + kibanaVersion: deps.kibanaVersion, + }; // Wait for all requests to complete, in case one of them fails return the successfull ones const results = await Promise.allSettled([ - getMappings(settings, legacyConfig), - getAliases(settings, legacyConfig), - getDataStreams(settings, legacyConfig), - getLegacyTemplates(settings, legacyConfig), - getIndexTemplates(settings, legacyConfig), - getComponentTemplates(settings, legacyConfig), + getMappings(settings, configWithHeaders), + getAliases(settings, configWithHeaders), + getDataStreams(settings, configWithHeaders), + getLegacyTemplates(settings, configWithHeaders), + getIndexTemplates(settings, configWithHeaders), + getComponentTemplates(settings, configWithHeaders), ]); const [mappings, aliases, dataStreams, legacyTemplates, indexTemplates, componentTemplates] = diff --git a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts index eddefc86fcbd2..70de8d6c59c7f 100644 --- a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts +++ b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts @@ -43,7 +43,7 @@ function filterHeaders(originalHeaders: object, headersToKeep: string[]): object return pick(originalHeaders, headersToKeepNormalized); } -function getRequestConfig( +export function getRequestConfig( headers: object, esConfig: ESConfigForProxy, uri: string, diff --git a/test/api_integration/apis/console/index.ts b/test/api_integration/apis/console/index.ts index ad4f8256f97ad..81f6f17f77b87 100644 --- a/test/api_integration/apis/console/index.ts +++ b/test/api_integration/apis/console/index.ts @@ -11,5 +11,6 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('core', () => { loadTestFile(require.resolve('./proxy_route')); + loadTestFile(require.resolve('./autocomplete_entities')); }); } From 0a32fe89683055cd47a3acf450ec2c3be1ae75fa Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 3 Nov 2022 17:14:37 +0500 Subject: [PATCH 14/17] Restore integration tests --- .../apis/console/autocomplete_entities.ts | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 test/api_integration/apis/console/autocomplete_entities.ts diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts new file mode 100644 index 0000000000000..7f74156f379a0 --- /dev/null +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import type { Response } from 'superagent'; +import type { FtrProviderContext } from '../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + + function utilTest(name: string, query: object, test: (response: Response) => void) { + it(name, async () => { + const response = await supertest.get('/api/console/autocomplete_entities').query(query); + test(response); + }); + } + + describe('/api/console/autocomplete_entities', () => { + utilTest('should not succeed if no settings are provided in query params', {}, (response) => { + const { status } = response; + expect(status).to.be(400); + }); + + utilTest( + 'should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', + { + indices: true, + fields: true, + templates: true, + dataStreams: true, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body).sort()).to.eql([ + 'aliases', + 'componentTemplates', + 'dataStreams', + 'indexTemplates', + 'legacyTemplates', + 'mappings', + ]); + } + ); + + utilTest( + 'should return empty payload with all settings are set to false', + { + indices: false, + fields: false, + templates: false, + dataStreams: false, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + expect(body.aliases).to.eql({}); + expect(body.mappings).to.eql({}); + expect(body.dataStreams).to.eql({}); + } + ); + + utilTest( + 'should return empty templates with templates setting is set to false', + { + indices: true, + fields: true, + templates: false, + dataStreams: true, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + } + ); + + utilTest( + 'should return empty data streams with dataStreams setting is set to false', + { + indices: true, + fields: true, + templates: true, + dataStreams: false, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(body.dataStreams).to.eql({}); + } + ); + + utilTest( + 'should return empty aliases with indices setting is set to false', + { + indices: false, + fields: true, + templates: true, + dataStreams: true, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(body.aliases).to.eql({}); + } + ); + + utilTest( + 'should return empty mappings with fields setting is set to false', + { + indices: true, + fields: false, + templates: true, + dataStreams: true, + }, + (response) => { + const { body, status } = response; + expect(status).to.be(200); + expect(body.mappings).to.eql({}); + } + ); + }); +}; From f94b90bec283882fb77bbac887a4f469e02ada61 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 10 Nov 2022 14:06:01 +0500 Subject: [PATCH 15/17] Address CR changes and expand api integration tests --- .../console/autocomplete_entities/index.ts | 35 ++- .../apis/console/autocomplete_entities.ts | 226 ++++++++++++++++++ 2 files changed, 250 insertions(+), 11 deletions(-) diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index 49fbdb9a3659a..1c6b23fa7fd7f 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -18,7 +18,7 @@ import { sanitizeHostname } from '../../../../lib/utils'; import type { ESConfigForProxy } from '../../../../types'; import { getRequestConfig } from '../proxy/create_handler'; -interface Settings { +interface SettingsToRetrieve { indices: boolean; fields: boolean; templates: boolean; @@ -31,7 +31,7 @@ const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB // Limit the response size to 10MB, because the response can be very large and sending it to the client // can cause the browser to hang. -const getMappings = async (settings: Settings, config: Config) => { +const getMappings = async (settings: SettingsToRetrieve, config: Config) => { if (settings.fields) { const mappings = await getEntity('/_mapping', config); return mappings; @@ -40,7 +40,7 @@ const getMappings = async (settings: Settings, config: Config) => { return {}; }; -const getAliases = async (settings: Settings, config: Config) => { +const getAliases = async (settings: SettingsToRetrieve, config: Config) => { if (settings.indices) { const aliases = await getEntity('/_alias', config); return aliases; @@ -49,7 +49,7 @@ const getAliases = async (settings: Settings, config: Config) => { return {}; }; -const getDataStreams = async (settings: Settings, config: Config) => { +const getDataStreams = async (settings: SettingsToRetrieve, config: Config) => { if (settings.dataStreams) { const dataStreams = await getEntity('/_data_stream', config); return dataStreams; @@ -58,7 +58,7 @@ const getDataStreams = async (settings: Settings, config: Config) => { return {}; }; -const getLegacyTemplates = async (settings: Settings, config: Config) => { +const getLegacyTemplates = async (settings: SettingsToRetrieve, config: Config) => { if (settings.templates) { const legacyTemplates = await getEntity('/_template', config); return legacyTemplates; @@ -67,7 +67,7 @@ const getLegacyTemplates = async (settings: Settings, config: Config) => { return {}; }; -const getIndexTemplates = async (settings: Settings, config: Config) => { +const getIndexTemplates = async (settings: SettingsToRetrieve, config: Config) => { if (settings.templates) { const indexTemplates = await getEntity('/_index_template', config); return indexTemplates; @@ -76,7 +76,7 @@ const getIndexTemplates = async (settings: Settings, config: Config) => { return {}; }; -const getComponentTemplates = async (settings: Settings, config: Config) => { +const getComponentTemplates = async (settings: SettingsToRetrieve, config: Config) => { if (settings.templates) { const componentTemplates = await getEntity('/_component_template', config); return componentTemplates; @@ -85,6 +85,17 @@ const getComponentTemplates = async (settings: Settings, config: Config) => { return {}; }; +/** + * Get the autocomplete suggestions for the given entity. + * We are using the raw http request in this function to retrieve the entities instead of esClient because + * the esClient does not handle large responses well. For example, the response size for + * the mappings can be very large(> 1GB) and the esClient will throw an 'Invalid string length' + * error when trying to parse the response. By using the raw http request, we can limit the + * response size and avoid the error. + * @param path The path to the entity to retrieve. For example, '/_mapping' or '/_alias'. + * @param config The configuration for the request. + * @returns The entity retrieved from Elasticsearch. + */ const getEntity = (path: string, config: Config) => { return new Promise((resolve, reject) => { const { hosts, kibanaVersion } = config; @@ -143,12 +154,14 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { validate: false, }, async (context, request, response) => { - const settings = parse(request.url.search, { parseBooleans: true }) as unknown as Settings; + const settings = parse(request.url.search, { + parseBooleans: true, + }) as unknown as SettingsToRetrieve; - // If no settings are provided return 400 + // If no settings are specified, then return 400. if (Object.keys(settings).length === 0) { return response.badRequest({ - body: 'Request must contain a query param of autocomplete settings', + body: 'Request must contain at least one of the following parameters: indices, fields, templates, dataStreams', }); } @@ -178,7 +191,7 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { // If the request failed, log the error and return an empty object if (result.reason instanceof Error) { - deps.log.error(result.reason); + deps.log.debug(`Failed to retrieve autocomplete suggestions: ${result.reason.message}`); } return {}; diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts index 7f74156f379a0..8e09ab16aab42 100644 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -13,6 +13,149 @@ import type { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const createIndex = async (indexName: string) => { + await supertest + .post(`/api/console/proxy?method=PUT&path=/${indexName}`) + .set('kbn-xsrf', 'true') + .send({ + mappings: { + properties: { + foo: { + type: 'text', + }, + }, + }, + }) + .expect(200); + }; + + const createAlias = async (indexName: string, aliasName: string) => { + await supertest + .post(`/api/console/proxy?method=POST&path=/_aliases`) + .set('kbn-xsrf', 'true') + .send({ + actions: [ + { + add: { + index: indexName, + alias: aliasName, + }, + }, + ], + }) + .expect(200); + }; + + const createLegacyTemplate = async (templateName: string) => { + await supertest + .post(`/api/console/proxy?method=PUT&path=/_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send({ + index_patterns: ['*'], + }); + }; + + const createComponentTemplate = async (templateName: string) => { + await supertest + .post(`/api/console/proxy?method=PUT&path=/_component_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send({ + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + format: 'date_optional_time||epoch_millis', + }, + message: { + type: 'wildcard', + }, + }, + }, + }, + _meta: { + description: 'Mappings for @timestamp and message fields', + 'my-custom-meta-field': 'More arbitrary metadata', + }, + }); + }; + + const createIndexTemplate = async ( + templateName: string, + indexPatterns: string[], + composedOf: string[] + ) => { + await supertest + .post(`/api/console/proxy?method=PUT&path=/_index_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send({ + index_patterns: indexPatterns, + data_stream: {}, + composed_of: composedOf, + priority: 500, + _meta: { + description: 'Template for my time series data', + 'my-custom-meta-field': 'More arbitrary metadata', + }, + }) + .expect(200); + }; + + const createDataStream = async (dataStream: string) => { + await supertest + .post(`/api/console/proxy?method=PUT&path=/_data_stream/${dataStream}`) + .set('kbn-xsrf', 'true') + .send(); + }; + + const deleteIndex = async (indexName: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/${indexName}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + + const deleteAlias = async (indexName: string, aliasName: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/${indexName}/_alias/${aliasName}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + + const deleteIndexTemplate = async (templateName: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/_index_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + + const deleteComponentTemplate = async (templateName: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/_component_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + + const deleteLegacyTemplate = async (templateName: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/_template/${templateName}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + + const deleteDataStream = async (dataStream: string) => { + await supertest + .post(`/api/console/proxy?method=DELETE&path=/_data_stream/${dataStream}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }; + function utilTest(name: string, query: object, test: (response: Response) => void) { it(name, async () => { const response = await supertest.get('/api/console/autocomplete_entities').query(query); @@ -21,6 +164,31 @@ export default ({ getService }: FtrProviderContext) => { } describe('/api/console/autocomplete_entities', () => { + const indexName = 'test-index-1'; + const aliasName = 'test-alias-1'; + const indexTemplateName = 'test-index-template-1'; + const componentTemplateName = 'test-component-template-1'; + const dataStreamName = 'test-data-stream-1'; + const legacyTemplateName = 'test-legacy-template-1'; + + before(async () => { + await createIndex(indexName); + await createAlias(indexName, aliasName); + await createComponentTemplate(componentTemplateName); + await createIndexTemplate(indexTemplateName, [`${dataStreamName}*`], [componentTemplateName]); + await createDataStream(dataStreamName); + await createLegacyTemplate(legacyTemplateName); + }); + + after(async () => { + await deleteAlias(indexName, aliasName); + await deleteIndex(indexName); + await deleteDataStream(dataStreamName); + await deleteIndexTemplate(indexTemplateName); + await deleteComponentTemplate(componentTemplateName); + await deleteLegacyTemplate(legacyTemplateName); + }); + utilTest('should not succeed if no settings are provided in query params', {}, (response) => { const { status } = response; expect(status).to.be(400); @@ -129,5 +297,63 @@ export default ({ getService }: FtrProviderContext) => { expect(body.mappings).to.eql({}); } ); + + it('should return mappings with fields setting is set to true', async () => { + const response = await supertest.get('/api/console/autocomplete_entities').query({ + indices: false, + fields: true, + templates: false, + dataStreams: false, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body.mappings)).to.contain(indexName); + }); + + it('should return aliases with indices setting is set to true', async () => { + const response = await supertest.get('/api/console/autocomplete_entities').query({ + indices: true, + fields: false, + templates: false, + dataStreams: false, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.aliases[indexName].aliases).to.eql({ [aliasName]: {} }); + }); + + it('should return data streams with dataStreams setting is set to true', async () => { + const response = await supertest.get('/api/console/autocomplete_entities').query({ + indices: false, + fields: false, + templates: false, + dataStreams: true, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.dataStreams.data_streams.map((ds: any) => ds.name)).to.contain(dataStreamName); + }); + + it('should return all templates with templates setting is set to true', async () => { + const response = await supertest.get('/api/console/autocomplete_entities').query({ + indices: false, + fields: false, + templates: true, + dataStreams: false, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body.legacyTemplates)).to.contain(legacyTemplateName); + expect(body.indexTemplates.index_templates.map((it: any) => it.name)).to.contain( + indexTemplateName + ); + expect(body.componentTemplates.component_templates.map((ct: any) => ct.name)).to.contain( + componentTemplateName + ); + }); }); }; From a31e4068a93687dc714f459d7a2a40add9be18d7 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Thu, 10 Nov 2022 14:23:15 +0500 Subject: [PATCH 16/17] Add a unit test for sanitize_hostname --- .../lib/utils/sanitize_hostname.test.ts | 27 +++++++++++++++++++ .../apis/console/autocomplete_entities.ts | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 src/plugins/console/server/lib/utils/sanitize_hostname.test.ts diff --git a/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts b/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts new file mode 100644 index 0000000000000..987481ddfa5fd --- /dev/null +++ b/src/plugins/console/server/lib/utils/sanitize_hostname.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { sanitizeHostname } from './sanitize_hostname'; + +describe('sanitizeHostname', () => { + it('should remove leading and trailing brackets', () => { + expect(sanitizeHostname('[::1]')).toBe('::1'); + }); + + it('should remove leading brackets', () => { + expect(sanitizeHostname('[::1')).toBe('::1'); + }); + + it('should remove trailing brackets', () => { + expect(sanitizeHostname('::1]')).toBe('::1'); + }); + + it('should not remove brackets in the middle of the string', () => { + expect(sanitizeHostname('[::1]foo')).toBe('::1]foo'); + }); +}); diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts index 8e09ab16aab42..7bed0f8ec29bc 100644 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -175,7 +175,7 @@ export default ({ getService }: FtrProviderContext) => { await createIndex(indexName); await createAlias(indexName, aliasName); await createComponentTemplate(componentTemplateName); - await createIndexTemplate(indexTemplateName, [`${dataStreamName}*`], [componentTemplateName]); + await createIndexTemplate(indexTemplateName, [dataStreamName], [componentTemplateName]); await createDataStream(dataStreamName); await createLegacyTemplate(legacyTemplateName); }); From fc479a83da5b5b8beced8ffd0ac22009b07926c5 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Fri, 11 Nov 2022 12:10:58 +0500 Subject: [PATCH 17/17] Use es service to setup and cleanup, refactor tests --- .../apis/console/autocomplete_entities.ts | 334 +++++++----------- 1 file changed, 128 insertions(+), 206 deletions(-) diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts index 7bed0f8ec29bc..6bd899c979a2b 100644 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -7,17 +7,16 @@ */ import expect from '@kbn/expect'; -import type { Response } from 'superagent'; import type { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const client = getService('es'); const createIndex = async (indexName: string) => { - await supertest - .post(`/api/console/proxy?method=PUT&path=/${indexName}`) - .set('kbn-xsrf', 'true') - .send({ + await client.indices.create({ + index: indexName, + body: { mappings: { properties: { foo: { @@ -25,41 +24,30 @@ export default ({ getService }: FtrProviderContext) => { }, }, }, - }) - .expect(200); + }, + }); }; const createAlias = async (indexName: string, aliasName: string) => { - await supertest - .post(`/api/console/proxy?method=POST&path=/_aliases`) - .set('kbn-xsrf', 'true') - .send({ - actions: [ - { - add: { - index: indexName, - alias: aliasName, - }, - }, - ], - }) - .expect(200); + await client.indices.putAlias({ + index: indexName, + name: aliasName, + }); }; const createLegacyTemplate = async (templateName: string) => { - await supertest - .post(`/api/console/proxy?method=PUT&path=/_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send({ + await client.indices.putTemplate({ + name: templateName, + body: { index_patterns: ['*'], - }); + }, + }); }; const createComponentTemplate = async (templateName: string) => { - await supertest - .post(`/api/console/proxy?method=PUT&path=/_component_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send({ + await client.cluster.putComponentTemplate({ + name: templateName, + body: { template: { mappings: { properties: { @@ -73,11 +61,8 @@ export default ({ getService }: FtrProviderContext) => { }, }, }, - _meta: { - description: 'Mappings for @timestamp and message fields', - 'my-custom-meta-field': 'More arbitrary metadata', - }, - }); + }, + }); }; const createIndexTemplate = async ( @@ -85,83 +70,63 @@ export default ({ getService }: FtrProviderContext) => { indexPatterns: string[], composedOf: string[] ) => { - await supertest - .post(`/api/console/proxy?method=PUT&path=/_index_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send({ + await client.indices.putIndexTemplate({ + name: templateName, + body: { index_patterns: indexPatterns, data_stream: {}, composed_of: composedOf, priority: 500, - _meta: { - description: 'Template for my time series data', - 'my-custom-meta-field': 'More arbitrary metadata', - }, - }) - .expect(200); + }, + }); }; const createDataStream = async (dataStream: string) => { - await supertest - .post(`/api/console/proxy?method=PUT&path=/_data_stream/${dataStream}`) - .set('kbn-xsrf', 'true') - .send(); + await client.indices.createDataStream({ + name: dataStream, + }); }; const deleteIndex = async (indexName: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/${indexName}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.indices.delete({ + index: indexName, + }); }; const deleteAlias = async (indexName: string, aliasName: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/${indexName}/_alias/${aliasName}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.indices.deleteAlias({ + index: indexName, + name: aliasName, + }); }; const deleteIndexTemplate = async (templateName: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/_index_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.indices.deleteIndexTemplate({ + name: templateName, + }); }; const deleteComponentTemplate = async (templateName: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/_component_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.cluster.deleteComponentTemplate({ + name: templateName, + }); }; const deleteLegacyTemplate = async (templateName: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/_template/${templateName}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.indices.deleteTemplate({ + name: templateName, + }); }; const deleteDataStream = async (dataStream: string) => { - await supertest - .post(`/api/console/proxy?method=DELETE&path=/_data_stream/${dataStream}`) - .set('kbn-xsrf', 'true') - .send() - .expect(200); + await client.indices.deleteDataStream({ + name: dataStream, + }); }; - function utilTest(name: string, query: object, test: (response: Response) => void) { - it(name, async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query(query); - test(response); - }); - } + const sendRequest = async (query: object) => { + return await supertest.get('/api/console/autocomplete_entities').query(query); + }; describe('/api/console/autocomplete_entities', () => { const indexName = 'test-index-1'; @@ -172,6 +137,7 @@ export default ({ getService }: FtrProviderContext) => { const legacyTemplateName = 'test-legacy-template-1'; before(async () => { + // Setup indices, aliases, templates, and data streams await createIndex(indexName); await createAlias(indexName, aliasName); await createComponentTemplate(componentTemplateName); @@ -181,6 +147,7 @@ export default ({ getService }: FtrProviderContext) => { }); after(async () => { + // Cleanup indices, aliases, templates, and data streams await deleteAlias(indexName, aliasName); await deleteIndex(indexName); await deleteDataStream(dataStreamName); @@ -189,122 +156,90 @@ export default ({ getService }: FtrProviderContext) => { await deleteLegacyTemplate(legacyTemplateName); }); - utilTest('should not succeed if no settings are provided in query params', {}, (response) => { + it('should not succeed if no settings are provided in query params', async () => { + const response = await sendRequest({}); const { status } = response; expect(status).to.be(400); }); - utilTest( - 'should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', - { + it('should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', async () => { + const response = await sendRequest({ indices: true, fields: true, templates: true, dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(Object.keys(body).sort()).to.eql([ - 'aliases', - 'componentTemplates', - 'dataStreams', - 'indexTemplates', - 'legacyTemplates', - 'mappings', - ]); - } - ); - - utilTest( - 'should return empty payload with all settings are set to false', - { + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body).sort()).to.eql([ + 'aliases', + 'componentTemplates', + 'dataStreams', + 'indexTemplates', + 'legacyTemplates', + 'mappings', + ]); + }); + + it('should return empty payload with all settings are set to false', async () => { + const response = await sendRequest({ indices: false, fields: false, templates: false, dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - expect(body.aliases).to.eql({}); - expect(body.mappings).to.eql({}); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty templates with templates setting is set to false', - { - indices: true, - fields: true, + }); + + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + expect(body.aliases).to.eql({}); + expect(body.mappings).to.eql({}); + expect(body.dataStreams).to.eql({}); + }); + + it('should return empty templates with templates setting is set to false', async () => { + const response = await sendRequest({ templates: false, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.legacyTemplates).to.eql({}); - expect(body.indexTemplates).to.eql({}); - expect(body.componentTemplates).to.eql({}); - } - ); - - utilTest( - 'should return empty data streams with dataStreams setting is set to false', - { - indices: true, - fields: true, - templates: true, + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.legacyTemplates).to.eql({}); + expect(body.indexTemplates).to.eql({}); + expect(body.componentTemplates).to.eql({}); + }); + + it('should return empty data streams with dataStreams setting is set to false', async () => { + const response = await sendRequest({ dataStreams: false, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.dataStreams).to.eql({}); - } - ); - - utilTest( - 'should return empty aliases with indices setting is set to false', - { + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.dataStreams).to.eql({}); + }); + + it('should return empty aliases with indices setting is set to false', async () => { + const response = await sendRequest({ indices: false, - fields: true, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.aliases).to.eql({}); - } - ); - - utilTest( - 'should return empty mappings with fields setting is set to false', - { - indices: true, + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.aliases).to.eql({}); + }); + + it('should return empty mappings with fields setting is set to false', async () => { + const response = await sendRequest({ fields: false, - templates: true, - dataStreams: true, - }, - (response) => { - const { body, status } = response; - expect(status).to.be(200); - expect(body.mappings).to.eql({}); - } - ); + }); + const { body, status } = response; + expect(status).to.be(200); + expect(body.mappings).to.eql({}); + }); it('should return mappings with fields setting is set to true', async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query({ - indices: false, - fields: true, - templates: false, - dataStreams: false, - }); + const response = await sendRequest({ fields: true }); const { body, status } = response; expect(status).to.be(200); @@ -312,12 +247,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should return aliases with indices setting is set to true', async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query({ - indices: true, - fields: false, - templates: false, - dataStreams: false, - }); + const response = await sendRequest({ indices: true }); const { body, status } = response; expect(status).to.be(200); @@ -325,35 +255,27 @@ export default ({ getService }: FtrProviderContext) => { }); it('should return data streams with dataStreams setting is set to true', async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query({ - indices: false, - fields: false, - templates: false, - dataStreams: true, - }); + const response = await sendRequest({ dataStreams: true }); const { body, status } = response; expect(status).to.be(200); - expect(body.dataStreams.data_streams.map((ds: any) => ds.name)).to.contain(dataStreamName); + expect(body.dataStreams.data_streams.map((ds: { name: string }) => ds.name)).to.contain( + dataStreamName + ); }); it('should return all templates with templates setting is set to true', async () => { - const response = await supertest.get('/api/console/autocomplete_entities').query({ - indices: false, - fields: false, - templates: true, - dataStreams: false, - }); + const response = await sendRequest({ templates: true }); const { body, status } = response; expect(status).to.be(200); expect(Object.keys(body.legacyTemplates)).to.contain(legacyTemplateName); - expect(body.indexTemplates.index_templates.map((it: any) => it.name)).to.contain( + expect(body.indexTemplates.index_templates.map((it: { name: string }) => it.name)).to.contain( indexTemplateName ); - expect(body.componentTemplates.component_templates.map((ct: any) => ct.name)).to.contain( - componentTemplateName - ); + expect( + body.componentTemplates.component_templates.map((ct: { name: string }) => ct.name) + ).to.contain(componentTemplateName); }); }); };