From 173b1b980dc3f4c8a70e9bcd1bce91d947e0569a Mon Sep 17 00:00:00 2001 From: Liza K Date: Mon, 5 Apr 2021 21:50:16 +0300 Subject: [PATCH 01/36] Move inspector adapter integration into search source --- .../search_examples/public/search/app.tsx | 4 +- .../data/common/search/aggs/buckets/terms.ts | 49 ++++++-------- .../expressions/esaggs/request_handler.ts | 65 +++++++------------ .../common/search/expressions/utils/index.ts | 1 - .../data/common/search/search_source/index.ts | 1 + .../search/search_source/inspect/index.ts | 9 +++ .../inspect/inspector_stats.ts} | 4 +- .../search_source/search_source.test.ts | 64 +++++++++--------- .../search/search_source/search_source.ts | 21 ++++-- src/plugins/data/common/search/types.ts | 3 + src/plugins/data/public/index.ts | 3 - src/plugins/data/server/index.ts | 5 -- .../public/application/angular/discover.js | 21 ++---- .../embeddable/search_embeddable.ts | 29 +++------ .../discover/public/kibana_services.ts | 2 +- .../es_geo_grid_source/es_geo_grid_source.tsx | 2 +- .../es_search_source/es_search_source.tsx | 2 +- .../classes/sources/es_source/es_source.ts | 49 ++++---------- .../generate_csv/generate_csv.ts | 2 +- 19 files changed, 143 insertions(+), 193 deletions(-) create mode 100644 src/plugins/data/common/search/search_source/inspect/index.ts rename src/plugins/data/common/search/{expressions/utils/courier_inspector_stats.ts => search_source/inspect/inspector_stats.ts} (97%) diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index c87bf21e0e71c..3bac445581ae7 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -204,8 +204,8 @@ export const SearchExamplesApp = ({ }); } - setRequest(await searchSource.getSearchRequestBody()); - const res = await searchSource.fetch(); + setRequest(searchSource.getSearchRequestBody()); + const res = await searchSource.fetch$().toPromise(); setResponse(res); const message = Searched {res.hits.total} documents.; diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 7d37dc83405b8..d4a6fe3a68916 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -21,7 +21,6 @@ import { aggTermsFnName } from './terms_fn'; import { AggConfigSerialized, BaseAggParams } from '../types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { getRequestInspectorStats, getResponseInspectorStats } from '../../expressions'; import { buildOtherBucketAgg, @@ -103,36 +102,28 @@ export const getTermsBucketAgg = () => nestedSearchSource.setField('aggs', filterAgg); - let request: ReturnType | undefined; - if (inspectorRequestAdapter) { - request = inspectorRequestAdapter.start( - i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { - defaultMessage: 'Other bucket', + const requestResponder = inspectorRequestAdapter?.start( + i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { + defaultMessage: 'Other bucket', + }), + { + description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { + defaultMessage: + 'This request counts the number of documents that fall ' + + 'outside the criterion of the data buckets.', }), - { - description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { - defaultMessage: - 'This request counts the number of documents that fall ' + - 'outside the criterion of the data buckets.', - }), - searchSessionId, - } - ); - nestedSearchSource.getSearchRequestBody().then((body) => { - request!.json(body); - }); - request.stats(getRequestInspectorStats(nestedSearchSource)); - } + searchSessionId, + } + ); + + const response = await nestedSearchSource + .fetch$({ + abortSignal, + sessionId: searchSessionId, + requestResponder, + }) + .toPromise(); - const response = await nestedSearchSource.fetch({ - abortSignal, - sessionId: searchSessionId, - }); - if (request) { - request - .stats(getResponseInspectorStats(response, nestedSearchSource)) - .ok({ json: response }); - } resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg()); } if (aggConfig.params.missingBucket) { diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 72d9cc4095570..5620698a47538 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -22,7 +22,6 @@ import { import { IAggConfigs } from '../../aggs'; import { ISearchStartSearchSource } from '../../search_source'; import { tabifyAggResponse } from '../../tabify'; -import { getRequestInspectorStats, getResponseInspectorStats } from '../utils'; /** @internal */ export interface RequestHandlerParams { @@ -41,6 +40,21 @@ export interface RequestHandlerParams { getNow?: () => Date; } +function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) { + return inspectorAdapters.requests?.start( + i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { + defaultMessage: 'Data', + }), + { + description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the visualization.', + }), + searchSessionId, + } + ); +} + export const handleRequest = async ({ abortSignal, aggs, @@ -113,52 +127,19 @@ export const handleRequest = async ({ requestSearchSource.setField('filter', filters); requestSearchSource.setField('query', query); - let request; - if (inspectorAdapters.requests) { - inspectorAdapters.requests.reset(); - request = inspectorAdapters.requests.start( - i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { - defaultMessage: 'Data', - }), - { - description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { - defaultMessage: - 'This request queries Elasticsearch to fetch the data for the visualization.', - }), - searchSessionId, - } - ); - request.stats(getRequestInspectorStats(requestSearchSource)); - } - - try { - const response = await requestSearchSource.fetch({ - abortSignal, - sessionId: searchSessionId, - }); - - if (request) { - request.stats(getResponseInspectorStats(response, searchSource)).ok({ json: response }); - } + inspectorAdapters.requests?.reset(); + const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId); - (searchSource as any).rawResponse = response; - } catch (e) { - // Log any error during request to the inspector - if (request) { - request.error({ json: e }); - } - throw e; - } finally { - // Add the request body no matter if things went fine or not - if (request) { - request.json(await requestSearchSource.getSearchRequestBody()); - } - } + const response$ = await requestSearchSource.fetch$({ + abortSignal, + sessionId: searchSessionId, + requestResponder, + }); // Note that rawResponse is not deeply cloned here, so downstream applications using courier // must take care not to mutate it, or it could have unintended side effects, e.g. displaying // response data incorrectly in the inspector. - let response = (searchSource as any).rawResponse; + let response = await response$.toPromise(); for (const agg of aggs.aggs) { if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { response = await agg.type.postFlightRequest( diff --git a/src/plugins/data/common/search/expressions/utils/index.ts b/src/plugins/data/common/search/expressions/utils/index.ts index 2fa54d47445b3..a6ea8da6ac6e9 100644 --- a/src/plugins/data/common/search/expressions/utils/index.ts +++ b/src/plugins/data/common/search/expressions/utils/index.ts @@ -6,5 +6,4 @@ * Side Public License, v 1. */ -export * from './courier_inspector_stats'; export * from './function_wrapper'; diff --git a/src/plugins/data/common/search/search_source/index.ts b/src/plugins/data/common/search/search_source/index.ts index 1cb04075dad7a..757e0de6ecb49 100644 --- a/src/plugins/data/common/search/search_source/index.ts +++ b/src/plugins/data/common/search/search_source/index.ts @@ -10,6 +10,7 @@ export { createSearchSource } from './create_search_source'; export { injectReferences } from './inject_references'; export { extractReferences } from './extract_references'; export { parseSearchSourceJSON } from './parse_json'; +export { getResponseInspectorStats } from './inspect'; export * from './fetch'; export * from './legacy'; export * from './search_source'; diff --git a/src/plugins/data/common/search/search_source/inspect/index.ts b/src/plugins/data/common/search/search_source/inspect/index.ts new file mode 100644 index 0000000000000..d5947f8a18cc9 --- /dev/null +++ b/src/plugins/data/common/search/search_source/inspect/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './inspector_stats'; diff --git a/src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts similarity index 97% rename from src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts rename to src/plugins/data/common/search/search_source/inspect/inspector_stats.ts index 99acbce8935c4..24507a7e13058 100644 --- a/src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts +++ b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts @@ -15,8 +15,8 @@ import { i18n } from '@kbn/i18n'; import type { estypes } from '@elastic/elasticsearch'; -import { ISearchSource } from 'src/plugins/data/public'; -import { RequestStatistics } from 'src/plugins/inspector/common'; +import type { ISearchSource } from 'src/plugins/data/public'; +import type { RequestStatistics } from 'src/plugins/inspector/common'; /** @public */ export function getRequestInspectorStats(searchSource: ISearchSource) { diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index fd97a3d3381a9..3726e5d0c33e8 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -125,7 +125,7 @@ describe('SearchSource', () => { }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['hello']); expect(request.script_fields).toEqual({ world: {} }); expect(request.fields).toEqual(['@timestamp']); @@ -144,7 +144,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['@timestamp']); searchSource.setField('fieldsFromSource', ['foo']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).not.toHaveProperty('docvalue_fields'); }); @@ -160,7 +160,7 @@ describe('SearchSource', () => { // @ts-expect-error TS won't like using this field name, but technically it's possible. searchSource.setField('docvalue_fields', ['world']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('docvalue_fields'); expect(request.docvalue_fields).toEqual(['world']); }); @@ -179,7 +179,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['c']); searchSource.setField('fieldsFromSource', ['a', 'b', 'd']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('docvalue_fields'); expect(request._source.includes).toEqual(['c', 'a', 'b', 'd']); expect(request.docvalue_fields).toEqual([{ field: 'b', format: 'date_time' }]); @@ -202,7 +202,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: 'hello', format: 'strict_date_time' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([{ field: 'hello', format: 'strict_date_time' }]); }); @@ -218,7 +218,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([{ field: 'hello', format: 'date_time' }]); }); @@ -239,7 +239,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: 'hello', a: 'a', c: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([ { field: 'hello', format: 'date_time', a: 'a', b: 'test', c: 'c' }, @@ -258,7 +258,7 @@ describe('SearchSource', () => { // @ts-expect-error TS won't like using this field name, but technically it's possible. searchSource.setField('script_fields', { world: {} }); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('script_fields'); expect(request.script_fields).toEqual({ hello: {}, @@ -277,7 +277,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'a', { field: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a', 'c']); }); @@ -293,7 +293,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'a', { foo: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a']); }); @@ -309,23 +309,23 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fieldsFromSource', ['hello', 'a']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a']); }); test('defaults to * for stored fields when no fields are provided', async () => { - const requestA = await searchSource.getSearchRequestBody(); + const requestA = searchSource.getSearchRequestBody(); expect(requestA.stored_fields).toEqual(['*']); searchSource.setField('fields', ['*']); - const requestB = await searchSource.getSearchRequestBody(); + const requestB = searchSource.getSearchRequestBody(); expect(requestB.stored_fields).toEqual(['*']); }); test('defaults to * for stored fields when no fields are provided with fieldsFromSource', async () => { searchSource.setField('fieldsFromSource', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['*']); }); }); @@ -343,7 +343,7 @@ describe('SearchSource', () => { // @ts-expect-error Typings for excludes filters need to be fixed. searchSource.setField('source', { excludes: ['exclude-*'] }); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['@timestamp']); }); @@ -357,7 +357,7 @@ describe('SearchSource', () => { }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['@timestamp']); }); @@ -372,7 +372,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); }); @@ -387,7 +387,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'foo']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['hello']); }); @@ -402,7 +402,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]); }); @@ -417,7 +417,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]); }); @@ -432,7 +432,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['timestamp', '*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {}, world: {} }); }); }); @@ -455,7 +455,7 @@ describe('SearchSource', () => { 'bar-b', ]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar-b'], }); @@ -473,7 +473,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['hello', '@timestamp', 'bar', 'date']); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['@timestamp', 'bar']); @@ -498,7 +498,7 @@ describe('SearchSource', () => { 'runtime_field', ]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar'], }); @@ -520,7 +520,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']); searchSource.setField('fieldsFromSource', ['foo-b', 'date', 'baz']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar', 'date', 'baz'], }); @@ -546,7 +546,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([ '*', { field: '@timestamp', format: 'strict_date_optional_time_nanos' }, @@ -574,7 +574,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([ { field: 'foo-bar' }, { field: 'field1' }, @@ -592,14 +592,14 @@ describe('SearchSource', () => { expect(searchSource.getField('source')).toBe(undefined); searchSource.setField('index', indexPattern); expect(searchSource.getField('index')).toBe(indexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(mockSource); }); test('removes created searchSource filter on removal', async () => { searchSource.setField('index', indexPattern); searchSource.setField('index', undefined); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(undefined); }); }); @@ -609,7 +609,7 @@ describe('SearchSource', () => { searchSource.setField('index', indexPattern); searchSource.setField('index', indexPattern2); expect(searchSource.getField('index')).toBe(indexPattern2); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(mockSource2); }); @@ -617,7 +617,7 @@ describe('SearchSource', () => { searchSource.setField('index', indexPattern); searchSource.setField('index', indexPattern2); searchSource.setField('index', undefined); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(undefined); }); }); @@ -808,7 +808,7 @@ describe('SearchSource', () => { docvalueFields: [], }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['geometry', 'prop1']); expect(request.docvalue_fields).toEqual(['prop1']); expect(request._source).toEqual(['geometry']); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index f11e7f06b6ab9..e1e7a8292d677 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -60,7 +60,7 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; -import { map, switchMap, tap } from 'rxjs/operators'; +import { catchError, finalize, map, switchMap, tap } from 'rxjs/operators'; import { defer, from } from 'rxjs'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; @@ -73,6 +73,7 @@ import type { SearchSourceFields, } from './types'; import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch'; +import { getRequestInspectorStats, getResponseInspectorStats } from './inspect'; import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; @@ -256,6 +257,9 @@ export class SearchSource { fetch$(options: ISearchOptions = {}) { const { getConfig } = this.dependencies; return defer(() => this.requestIsStarting(options)).pipe( + tap(() => { + options.requestResponder?.stats(getRequestInspectorStats(this)); + }), switchMap(() => { const searchRequest = this.flatten(); this.history = [searchRequest]; @@ -271,7 +275,17 @@ export class SearchSource { // TODO: Remove casting when https://github.com/elastic/elasticsearch-js/issues/1287 is resolved if ((response as any).error) { throw new RequestFailure(null, response); + } else { + options.requestResponder?.stats(getResponseInspectorStats(response, this)); + options.requestResponder?.ok({ json: response }); } + }), + catchError((e) => { + options.requestResponder?.error({ json: e }); + throw e; + }), + finalize(() => { + options.requestResponder?.json(this.getSearchRequestBody()); }) ); } @@ -298,9 +312,8 @@ export class SearchSource { /** * Returns body contents of the search request, often referred as query DSL. */ - async getSearchRequestBody() { - const searchRequest = await this.flatten(); - return searchRequest.body; + getSearchRequestBody() { + return this.flatten().body; } /** diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index d77a2ea62bb9a..37de8dc49d3c6 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -9,6 +9,7 @@ import { Observable } from 'rxjs'; import { IEsSearchRequest, IEsSearchResponse } from './es_search'; import { IndexPattern } from '..'; +import type { RequestResponder } from '../../../inspector/common'; export type ISearchGeneric = < SearchStrategyRequest extends IKibanaSearchRequest = IEsSearchRequest, @@ -118,6 +119,8 @@ export interface ISearchOptions { */ indexPattern?: IndexPattern; + + requestResponder?: RequestResponder; } /** diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index c47cd6cd9740d..8e531aae32407 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -314,8 +314,6 @@ import { boundsDescendingRaw, getNumberHistogramIntervalByDatatableColumn, getDateHistogramMetaDataByDatatableColumn, - // expressions utils - getRequestInspectorStats, getResponseInspectorStats, // tabify tabifyAggResponse, @@ -426,7 +424,6 @@ export const search = { getNumberHistogramIntervalByDatatableColumn, getDateHistogramMetaDataByDatatableColumn, }, - getRequestInspectorStats, getResponseInspectorStats, tabifyAggResponse, tabifyGetColumns, diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index cbf09ef57d96a..fa54f45d2feb2 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -176,9 +176,6 @@ import { parseEsInterval, parseInterval, toAbsoluteDates, - // expressions utils - getRequestInspectorStats, - getResponseInspectorStats, // tabify tabifyAggResponse, tabifyGetColumns, @@ -263,8 +260,6 @@ export const search = { toAbsoluteDates, calcAutoIntervalLessThan, }, - getRequestInspectorStats, - getResponseInspectorStats, tabifyAggResponse, tabifyGetColumns, }; diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 2c80fc111c740..0bf97365d44fe 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -25,8 +25,6 @@ import { discoverResponseHandler } from './response_handler'; import { getAngularModule, getHeaderActionMenuMounter, - getRequestInspectorStats, - getResponseInspectorStats, getServices, getUrlTracker, redirectWhenMissing, @@ -153,7 +151,6 @@ function discoverController($route, $scope) { const subscriptions = new Subscription(); const refetch$ = new Subject(); - let inspectorRequest; let isChangingIndexPattern = false; const savedSearch = $route.current.locals.savedObjects.savedSearch; const persistentSearchSource = savedSearch.searchSource; @@ -418,12 +415,14 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.LOADING; $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); - logInspectorRequest({ searchSessionId }); + return $scope.volatileSearchSource - .fetch({ + .fetch$({ abortSignal: abortController.signal, sessionId: searchSessionId, + requestResponder: getRequestResponder({ searchSessionId }), }) + .toPromise() .then(onResults) .catch((error) => { // If the request was aborted then no need to surface this error in the UI @@ -440,10 +439,6 @@ function discoverController($route, $scope) { }; function onResults(resp) { - inspectorRequest - .stats(getResponseInspectorStats(resp, $scope.volatileSearchSource)) - .ok({ json: resp }); - if (getTimeField() && !$scope.state.hideChart) { const tabifiedData = tabifyAggResponse($scope.opts.chartAggConfigs, resp); $scope.volatileSearchSource.rawResponse = resp; @@ -464,7 +459,7 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.COMPLETE; } - function logInspectorRequest({ searchSessionId = null } = { searchSessionId: null }) { + function getRequestResponder({ searchSessionId = null } = { searchSessionId: null }) { inspectorAdapters.requests.reset(); const title = i18n.translate('discover.inspectorRequestDataTitle', { defaultMessage: 'data', @@ -472,11 +467,7 @@ function discoverController($route, $scope) { const description = i18n.translate('discover.inspectorRequestDescription', { defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', }); - inspectorRequest = inspectorAdapters.requests.start(title, { description, searchSessionId }); - inspectorRequest.stats(getRequestInspectorStats($scope.volatileSearchSource)); - $scope.volatileSearchSource.getSearchRequestBody().then((body) => { - inspectorRequest.json(body); - }); + return inspectorAdapters.requests.start(title, { description, searchSessionId }); } $scope.resetQuery = function () { diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index e7349ed22355a..237da72ae3a52 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -29,13 +29,7 @@ import searchTemplateGrid from './search_template_datagrid.html'; import { ISearchEmbeddable, SearchInput, SearchOutput } from './types'; import { SortOrder } from '../angular/doc_table/components/table_header/helpers'; import { getSortForSearchSource } from '../angular/doc_table'; -import { - getRequestInspectorStats, - getResponseInspectorStats, - getServices, - IndexPattern, - ISearchSource, -} from '../../kibana_services'; +import { getServices, IndexPattern, ISearchSource } from '../../kibana_services'; import { SEARCH_EMBEDDABLE_TYPE } from './constants'; import { SavedSearch } from '../..'; import { @@ -330,14 +324,11 @@ export class SearchEmbeddable defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', }); - const inspectorRequest = this.inspectorAdapters.requests!.start(title, { + const requestResponder = this.inspectorAdapters.requests!.start(title, { description, searchSessionId, }); - inspectorRequest.stats(getRequestInspectorStats(searchSource)); - searchSource.getSearchRequestBody().then((body: Record) => { - inspectorRequest.json(body); - }); + this.searchScope.$apply(() => { this.searchScope!.isLoading = true; }); @@ -345,15 +336,15 @@ export class SearchEmbeddable try { // Make the request - const resp = await searchSource.fetch({ - abortSignal: this.abortController.signal, - sessionId: searchSessionId, - }); + const resp = await searchSource + .fetch$({ + abortSignal: this.abortController.signal, + sessionId: searchSessionId, + requestResponder, + }) + .toPromise(); this.updateOutput({ loading: false, error: undefined }); - // Log response to inspector - inspectorRequest.stats(getResponseInspectorStats(resp, searchSource)).ok({ json: resp }); - // Apply the changes to the angular scope this.searchScope.$apply(() => { this.searchScope!.hits = resp.hits.hits; diff --git a/src/plugins/discover/public/kibana_services.ts b/src/plugins/discover/public/kibana_services.ts index 27bcc00234939..e4b0035ed0e03 100644 --- a/src/plugins/discover/public/kibana_services.ts +++ b/src/plugins/discover/public/kibana_services.ts @@ -88,7 +88,7 @@ export const [getScopedHistory, setScopedHistory] = createGetterSetter abortController.abort()); - const inspectorAdapters = this.getInspectorAdapters(); - let inspectorRequest: RequestResponder | undefined; - if (inspectorAdapters?.requests) { - inspectorRequest = inspectorAdapters.requests.start(requestName, { - id: requestId, - description: requestDescription, - searchSessionId, - }); - } + const requestResponder = this.getInspectorAdapters()?.requests?.start(requestName, { + id: requestId, + description: requestDescription, + searchSessionId, + }); let resp; try { - if (inspectorRequest) { - const requestStats = search.getRequestInspectorStats(searchSource); - inspectorRequest.stats(requestStats); - searchSource.getSearchRequestBody().then((body) => { - if (inspectorRequest) { - inspectorRequest.json(body); - } - }); - } - resp = await searchSource.fetch({ - abortSignal: abortController.signal, - sessionId: searchSessionId, - legacyHitsTotal: false, - }); - if (inspectorRequest) { - const responseStats = search.getResponseInspectorStats(resp, searchSource); - inspectorRequest.stats(responseStats).ok({ json: resp }); - } + resp = await searchSource + .fetch$({ + abortSignal: abortController.signal, + sessionId: searchSessionId, + legacyHitsTotal: false, + requestResponder, + }) + .toPromise(); } catch (error) { - if (inspectorRequest) { - inspectorRequest.error(error); - } if (isSearchSourceAbortError(error)) { throw new DataRequestAbortError(); } diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 85c5379a63b7f..01959ed08036d 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -79,7 +79,7 @@ export class CsvGenerator { searchSource: ISearchSource, scrollSettings: CsvExportSettings['scroll'] ) { - const searchBody = await searchSource.getSearchRequestBody(); + const searchBody = searchSource.getSearchRequestBody(); this.logger.debug(`executing search request`); const searchParams = { params: { From d3c9dbb9fc46d51b0ab5efb5659edd3abbf35eae Mon Sep 17 00:00:00 2001 From: Liza K Date: Tue, 6 Apr 2021 11:29:19 +0300 Subject: [PATCH 02/36] docs and ts --- ...ugin-plugins-data-public.isearchoptions.md | 1 + ...-public.isearchoptions.requestresponder.md | 11 ++++++ ...ibana-plugin-plugins-data-public.search.md | 1 - ...ublic.searchsource.getsearchrequestbody.md | 4 +-- ...ugin-plugins-data-server.isearchoptions.md | 1 + ...-server.isearchoptions.requestresponder.md | 11 ++++++ ...ibana-plugin-plugins-data-server.search.md | 2 -- .../data/common/search/aggs/buckets/terms.ts | 1 - .../esaggs/request_handler.test.ts | 2 +- src/plugins/data/public/public.api.md | 36 ++++++++++--------- src/plugins/data/server/server.api.md | 33 +++++++++-------- 11 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md index 2473c9cfdde8d..cc0cb538be611 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md @@ -19,6 +19,7 @@ export interface ISearchOptions | [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | +| [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md new file mode 100644 index 0000000000000..b4431b9467b71 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) + +## ISearchOptions.requestResponder property + +Signature: + +```typescript +requestResponder?: RequestResponder; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md index cfaad01c029ea..259009c1c5668 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md @@ -53,7 +53,6 @@ search: { timeRange: import("../common").TimeRange | undefined; } | undefined; }; - getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md index cc50d3f017971..d384b9659dbcd 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md @@ -9,9 +9,9 @@ Returns body contents of the search request, often referred as query DSL. Signature: ```typescript -getSearchRequestBody(): Promise; +getSearchRequestBody(): any; ``` Returns: -`Promise` +`any` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md index 7fd4dd5b8e566..413a59be3d427 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md @@ -19,6 +19,7 @@ export interface ISearchOptions | [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | +| [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md new file mode 100644 index 0000000000000..7440f5a9d26cf --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) + +## ISearchOptions.requestResponder property + +Signature: + +```typescript +requestResponder?: RequestResponder; +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md index 0911c3e86964d..930f7710f9a00 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md @@ -36,8 +36,6 @@ search: { toAbsoluteDates: typeof toAbsoluteDates; calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; - getRequestInspectorStats: typeof getRequestInspectorStats; - getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; } diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index d4a6fe3a68916..77c9c6e391c0a 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -8,7 +8,6 @@ import { noop } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { RequestAdapter } from 'src/plugins/inspector/common'; import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts index 7580032b0dd85..c2566535916a8 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts @@ -133,7 +133,7 @@ describe('esaggs expression function - public', () => { test('calls searchSource.fetch', async () => { await handleRequest(mockParams); const searchSource = await mockParams.searchSourceService.create(); - expect(searchSource.fetch).toHaveBeenCalledWith({ + expect(searchSource.fetch$).toHaveBeenCalledWith({ abortSignal: mockParams.abortSignal, sessionId: mockParams.searchSessionId, }); diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 7f243cefd08b6..19ea754ed9a59 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1672,6 +1672,10 @@ export interface ISearchOptions { isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; + // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts + // + // (undocumented) + requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } @@ -2288,7 +2292,6 @@ export const search: { timeRange: import("../common").TimeRange | undefined; } | undefined; }; - getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; @@ -2424,7 +2427,7 @@ export class SearchSource { getId(): string; getOwnField(field: K): SearchSourceFields[K]; getParent(): SearchSource | undefined; - getSearchRequestBody(): Promise; + getSearchRequestBody(): any; getSerializedFields(recurse?: boolean): SearchSourceFields; // Warning: (ae-incompatible-release-tags) The symbol "history" is marked as @public, but its signature references "SearchRequest" which is marked as @internal // @@ -2690,21 +2693,20 @@ export const UI_SETTINGS: { // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:406:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:407:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:416:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:417:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:418:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:419:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:427:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:431:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:402:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:402:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:402:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:404:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:405:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:414:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:415:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:416:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:417:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:425:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:426:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:5 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts // src/plugins/data/public/search/session/session_service.ts:56:5 - (ae-forgotten-export) The symbol "UrlGeneratorStateMapping" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 29a5a67239171..6fda695a40c24 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -56,7 +56,6 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; -import { RequestStatistics } from 'src/plugins/inspector/common'; import { SavedObject } from 'kibana/server'; import { SavedObject as SavedObject_2 } from 'src/core/server'; import { SavedObjectsClientContract } from 'src/core/server'; @@ -1002,6 +1001,10 @@ export interface ISearchOptions { isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; + // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts + // + // (undocumented) + requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } @@ -1325,8 +1328,6 @@ export const search: { toAbsoluteDates: typeof toAbsoluteDates; calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; - getRequestInspectorStats: typeof getRequestInspectorStats; - getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; }; @@ -1508,20 +1509,18 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "HistogramFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:246:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:247:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:256:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:257:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:258:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:262:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:263:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:267:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:270:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:271:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:241:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:241:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:243:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:244:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:253:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:255:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:259:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:260:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:264:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:267:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:268:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/plugin.ts:81:74 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:114:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts From 413b462a890c31ee8c94b1b1ad5f73223a501dff Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 7 Apr 2021 13:07:20 +0300 Subject: [PATCH 03/36] Move other bucket to search source --- .../data/common/search/aggs/agg_type.ts | 31 +++--- .../_terms_other_bucket_helper.test.ts | 5 +- .../buckets/_terms_other_bucket_helper.ts | 9 +- .../data/common/search/aggs/buckets/terms.ts | 26 +++-- .../expressions/esaggs/request_handler.ts | 59 ++++------- .../search/search_source/search_source.ts | 98 +++++++++++++++++-- .../data/common/search/search_source/types.ts | 3 +- src/plugins/data/common/search/types.ts | 11 ++- .../public/application/angular/discover.js | 22 ++--- .../embeddable/search_embeddable.ts | 22 ++--- .../classes/sources/es_source/es_source.ts | 13 ++- 11 files changed, 176 insertions(+), 123 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index 33fdc45a605b7..f0f3912bf64fe 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -13,12 +13,23 @@ import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; import type { RequestAdapter } from 'src/plugins/inspector/common'; +import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; import { IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; +type PostFlightRequestFn = ( + resp: estypes.SearchResponse, + aggConfigs: IAggConfigs, + aggConfig: TAggConfig, + searchSource: ISearchSource, + inspectorRequestAdapter?: RequestAdapter, + abortSignal?: AbortSignal, + searchSessionId?: string +) => Promise>; + export interface AggTypeConfig< TAggConfig extends AggConfig = AggConfig, TParam extends AggParamType = AggParamType @@ -40,15 +51,7 @@ export interface AggTypeConfig< customLabels?: boolean; json?: boolean; decorateAggConfig?: () => any; - postFlightRequest?: ( - resp: any, - aggConfigs: IAggConfigs, - aggConfig: TAggConfig, - searchSource: ISearchSource, - inspectorRequestAdapter?: RequestAdapter, - abortSignal?: AbortSignal, - searchSessionId?: string - ) => Promise; + postFlightRequest?: PostFlightRequestFn; getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat; getValue?: (agg: TAggConfig, bucket: any) => any; getKey?: (bucket: any, key: any, agg: TAggConfig) => any; @@ -188,15 +191,7 @@ export class AggType< * @param searchSessionId - searchSessionId to be used for grouping requests into a single search session * @return {Promise} */ - postFlightRequest: ( - resp: any, - aggConfigs: IAggConfigs, - aggConfig: TAggConfig, - searchSource: ISearchSource, - inspectorRequestAdapter?: RequestAdapter, - abortSignal?: AbortSignal, - searchSessionId?: string - ) => Promise; + postFlightRequest: PostFlightRequestFn; /** * Get the serialized format for the values produced by this agg type, * overridden by several metrics that always output a simple number. diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 4e278d5872a3e..62cd603194250 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -16,6 +16,7 @@ import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; import { BUCKET_TYPES } from './bucket_agg_types'; import { IBucketAggConfig } from './bucket_agg_type'; import { mockAggTypesRegistry } from '../test_helpers'; +import { estypes } from '@elastic/elasticsearch'; const indexPattern = { id: '1234', @@ -416,7 +417,7 @@ describe('Terms Agg Other bucket helper', () => { aggConfigs.aggs[0] as IBucketAggConfig, otherAggConfig() ); - expect(mergedResponse.aggregations['1'].buckets[3].key).toEqual('__other__'); + expect((mergedResponse!.aggregations!['1'] as any).buckets[3].key).toEqual('__other__'); } }); @@ -438,7 +439,7 @@ describe('Terms Agg Other bucket helper', () => { otherAggConfig() ); - expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).toEqual( + expect((mergedResponse!.aggregations!['1'].buckets as any)[1]['2'].buckets[3].key).toEqual( '__other__' ); } diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 742615bc49d8f..576498d345f83 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -7,6 +7,7 @@ */ import { isNumber, keys, values, find, each, cloneDeep, flatten } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; import { buildExistsFilter, buildPhrasesFilter, buildQueryFromFilters } from '../../../../common'; import { AggGroupNames } from '../agg_groups'; import { IAggConfigs } from '../agg_configs'; @@ -42,7 +43,7 @@ const getNestedAggDSL = (aggNestedDsl: Record, startFromAggId: stri */ const getAggResultBuckets = ( aggConfigs: IAggConfigs, - response: any, + response: estypes.SearchResponse['aggregations'], aggWithOtherBucket: IBucketAggConfig, key: string ) => { @@ -235,11 +236,11 @@ export const buildOtherBucketAgg = ( export const mergeOtherBucketAggResponse = ( aggsConfig: IAggConfigs, - response: any, + response: estypes.SearchResponse, otherResponse: any, otherAgg: IBucketAggConfig, requestAgg: Record -) => { +): estypes.SearchResponse => { const updatedResponse = cloneDeep(response); each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { if (!bucket.doc_count || key === undefined) return; @@ -276,7 +277,7 @@ export const mergeOtherBucketAggResponse = ( }; export const updateMissingBucket = ( - response: any, + response: estypes.SearchResponse, aggConfigs: IAggConfigs, agg: IBucketAggConfig ) => { diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 77c9c6e391c0a..03cf14a577a50 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -101,25 +101,21 @@ export const getTermsBucketAgg = () => nestedSearchSource.setField('aggs', filterAgg); - const requestResponder = inspectorRequestAdapter?.start( - i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { - defaultMessage: 'Other bucket', - }), - { - description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { - defaultMessage: - 'This request counts the number of documents that fall ' + - 'outside the criterion of the data buckets.', - }), - searchSessionId, - } - ); - const response = await nestedSearchSource .fetch$({ abortSignal, sessionId: searchSessionId, - requestResponder, + inspector: { + adapter: inspectorRequestAdapter, + title: i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { + defaultMessage: 'Other bucket', + }), + description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { + defaultMessage: + 'This request counts the number of documents that fall ' + + 'outside the criterion of the data buckets.', + }), + }, }) .toPromise(); diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 5620698a47538..2f50a8daf08b3 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -40,21 +40,6 @@ export interface RequestHandlerParams { getNow?: () => Date; } -function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) { - return inspectorAdapters.requests?.start( - i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { - defaultMessage: 'Data', - }), - { - description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { - defaultMessage: - 'This request queries Elasticsearch to fetch the data for the visualization.', - }), - searchSessionId, - } - ); -} - export const handleRequest = async ({ abortSignal, aggs, @@ -100,9 +85,7 @@ export const handleRequest = async ({ }, }); - requestSearchSource.setField('aggs', function () { - return aggs.toDsl(metricsAtAllLevels); - }); + requestSearchSource.setField('aggs', aggs); requestSearchSource.onRequestStart((paramSearchSource, options) => { return aggs.onSearchRequestStart(paramSearchSource, options); @@ -128,31 +111,23 @@ export const handleRequest = async ({ requestSearchSource.setField('query', query); inspectorAdapters.requests?.reset(); - const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId); - const response$ = await requestSearchSource.fetch$({ - abortSignal, - sessionId: searchSessionId, - requestResponder, - }); - - // Note that rawResponse is not deeply cloned here, so downstream applications using courier - // must take care not to mutate it, or it could have unintended side effects, e.g. displaying - // response data incorrectly in the inspector. - let response = await response$.toPromise(); - for (const agg of aggs.aggs) { - if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { - response = await agg.type.postFlightRequest( - response, - aggs, - agg, - requestSearchSource, - inspectorAdapters.requests, - abortSignal, - searchSessionId - ); - } - } + const response = await requestSearchSource + .fetch$({ + abortSignal, + sessionId: searchSessionId, + inspector: { + adapter: inspectorAdapters.requests, + title: i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { + defaultMessage: 'Data', + }), + description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the visualization.', + }), + }, + }) + .toPromise(); const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null; const tabifyParams = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index e1e7a8292d677..9eb8b42fa1fbf 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -61,11 +61,12 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; import { catchError, finalize, map, switchMap, tap } from 'rxjs/operators'; -import { defer, from } from 'rxjs'; +import { defer, from, Observable } from 'rxjs'; +import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; -import { ISearchGeneric, ISearchOptions } from '../..'; +import { AggConfigs, ISearchGeneric, ISearchOptions } from '../..'; import type { ISearchSource, SearchFieldValue, @@ -75,7 +76,14 @@ import type { import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch'; import { getRequestInspectorStats, getResponseInspectorStats } from './inspect'; -import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common'; +import { + getEsQueryConfig, + buildEsQuery, + Filter, + UI_SETTINGS, + isCompleteResponse, + IKibanaSearchResponse, +} from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; import { fetchSoon } from './legacy'; import { extractReferences } from './extract_references'; @@ -256,9 +264,16 @@ export class SearchSource { */ fetch$(options: ISearchOptions = {}) { const { getConfig } = this.dependencies; + + const { id, title, description, adapter } = options.inspector || { title: '' }; + const requestResponder = adapter?.start(title, { + id, + description, + searchSessionId: options.sessionId, + }); return defer(() => this.requestIsStarting(options)).pipe( tap(() => { - options.requestResponder?.stats(getRequestInspectorStats(this)); + requestResponder?.stats(getRequestInspectorStats(this)); }), switchMap(() => { const searchRequest = this.flatten(); @@ -276,16 +291,16 @@ export class SearchSource { if ((response as any).error) { throw new RequestFailure(null, response); } else { - options.requestResponder?.stats(getResponseInspectorStats(response, this)); - options.requestResponder?.ok({ json: response }); + requestResponder?.stats(getResponseInspectorStats(response, this)); + requestResponder?.ok({ json: response }); } }), catchError((e) => { - options.requestResponder?.error({ json: e }); + requestResponder?.error({ json: e }); throw e; }), finalize(() => { - options.requestResponder?.json(this.getSearchRequestBody()); + requestResponder?.json(this.getSearchRequestBody()); }) ); } @@ -328,6 +343,37 @@ export class SearchSource { * PRIVATE APIS ******/ + private hasPostFlightRequests() { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + return aggs.aggs.some( + (agg) => agg.enabled && typeof agg.type.postFlightRequest === 'function' + ); + } else { + return false; + } + } + + private async fetchOthers(response: estypes.SearchResponse, options: ISearchOptions) { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + for (const agg of aggs.aggs) { + if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { + response = await agg.type.postFlightRequest( + response, + aggs, + agg, + this, + options.inspector?.adapter, + options.abortSignal, + options.sessionId + ); + } + } + return response; + } + } + /** * Run a search using the search service * @return {Promise>} @@ -340,6 +386,36 @@ export class SearchSource { }); return search({ params, indexType: searchRequest.indexType }, options).pipe( + switchMap((response) => { + return new Observable>((obs) => { + if (!isCompleteResponse(response)) { + obs.next(response); + } else { + if (!this.hasPostFlightRequests()) { + obs.next(response); + obs.complete(); + } else { + obs.next({ + ...response, + isPartial: true, + isRunning: true, + }); + const sub = from(this.fetchOthers(response.rawResponse, options)).subscribe({ + next: (responseWithOther) => { + obs.next({ + ...response, + rawResponse: responseWithOther, + }); + }, + complete: () => { + sub.unsubscribe(); + obs.complete(); + }, + }); + } + } + }); + }), map(({ rawResponse }) => onResponse(searchRequest, rawResponse)) ); } @@ -452,6 +528,12 @@ export class SearchSource { getConfig(UI_SETTINGS.SORT_OPTIONS) ); return addToBody(key, sort); + case 'aggs': + if ((val as any) instanceof AggConfigs) { + return addToBody('aggs', val.toDsl(true)); + } else { + return addToBody('aggs', val); + } default: return addToBody(key, val); } diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts index a178b38693d92..99f3f67a5e257 100644 --- a/src/plugins/data/common/search/search_source/types.ts +++ b/src/plugins/data/common/search/search_source/types.ts @@ -7,6 +7,7 @@ */ import { NameList } from 'elasticsearch'; +import { IAggConfigs } from 'src/plugins/data/public'; import { Query } from '../..'; import { Filter } from '../../es_query'; import { IndexPattern } from '../../index_patterns'; @@ -78,7 +79,7 @@ export interface SearchSourceFields { /** * {@link AggConfigs} */ - aggs?: any; + aggs?: object | IAggConfigs | (() => object); from?: number; size?: number; source?: NameList; diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index 37de8dc49d3c6..73805e05e6f39 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -9,7 +9,7 @@ import { Observable } from 'rxjs'; import { IEsSearchRequest, IEsSearchResponse } from './es_search'; import { IndexPattern } from '..'; -import type { RequestResponder } from '../../../inspector/common'; +import type { RequestAdapter } from '../../../inspector/common'; export type ISearchGeneric = < SearchStrategyRequest extends IKibanaSearchRequest = IEsSearchRequest, @@ -81,6 +81,13 @@ export interface IKibanaSearchRequest { params?: Params; } +export interface IInspectorInfo { + adapter?: RequestAdapter; + title: string; + id?: string; + description?: string; +} + export interface ISearchOptions { /** * An `AbortSignal` that allows the caller of `search` to abort a search request. @@ -120,7 +127,7 @@ export interface ISearchOptions { indexPattern?: IndexPattern; - requestResponder?: RequestResponder; + inspector?: IInspectorInfo; } /** diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 0bf97365d44fe..33533cd3d35cc 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -416,11 +416,20 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.LOADING; $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); + inspectorAdapters.requests.reset(); return $scope.volatileSearchSource .fetch$({ abortSignal: abortController.signal, sessionId: searchSessionId, - requestResponder: getRequestResponder({ searchSessionId }), + inspector: { + adapter: inspectorAdapters.requests, + title: i18n.translate('discover.inspectorRequestDataTitle', { + defaultMessage: 'data', + }), + description: i18n.translate('discover.inspectorRequestDescription', { + defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', + }), + }, }) .toPromise() .then(onResults) @@ -459,17 +468,6 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.COMPLETE; } - function getRequestResponder({ searchSessionId = null } = { searchSessionId: null }) { - inspectorAdapters.requests.reset(); - const title = i18n.translate('discover.inspectorRequestDataTitle', { - defaultMessage: 'data', - }); - const description = i18n.translate('discover.inspectorRequestDescription', { - defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', - }); - return inspectorAdapters.requests.start(title, { description, searchSessionId }); - } - $scope.resetQuery = function () { history.push( $route.current.params.id ? `/view/${encodeURIComponent($route.current.params.id)}` : '/' diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index 237da72ae3a52..dbaf07fed18c2 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -317,17 +317,6 @@ export class SearchEmbeddable // Log request to inspector this.inspectorAdapters.requests!.reset(); - const title = i18n.translate('discover.embeddable.inspectorRequestDataTitle', { - defaultMessage: 'Data', - }); - const description = i18n.translate('discover.embeddable.inspectorRequestDescription', { - defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', - }); - - const requestResponder = this.inspectorAdapters.requests!.start(title, { - description, - searchSessionId, - }); this.searchScope.$apply(() => { this.searchScope!.isLoading = true; @@ -340,7 +329,16 @@ export class SearchEmbeddable .fetch$({ abortSignal: this.abortController.signal, sessionId: searchSessionId, - requestResponder, + inspector: { + adapter: this.inspectorAdapters.requests, + title: i18n.translate('discover.embeddable.inspectorRequestDataTitle', { + defaultMessage: 'Data', + }), + description: i18n.translate('discover.embeddable.inspectorRequestDescription', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the search.', + }), + }, }) .toPromise(); this.updateOutput({ loading: false, error: undefined }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts index 2915eaec8ac77..50043772af95b 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts @@ -167,12 +167,6 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource const abortController = new AbortController(); registerCancelCallback(() => abortController.abort()); - const requestResponder = this.getInspectorAdapters()?.requests?.start(requestName, { - id: requestId, - description: requestDescription, - searchSessionId, - }); - let resp; try { resp = await searchSource @@ -180,7 +174,12 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource abortSignal: abortController.signal, sessionId: searchSessionId, legacyHitsTotal: false, - requestResponder, + inspector: { + adapter: this.getInspectorAdapters()?.requests, + id: requestId, + title: requestName, + description: requestDescription, + }, }) .toPromise(); } catch (error) { From 78624572cb98455127e99360fa67806e664374c4 Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 7 Apr 2021 16:30:36 +0300 Subject: [PATCH 04/36] test ts + delete unused tabilfy function --- ...s-data-public.isearchoptions.inspector.md} | 6 +-- ...ugin-plugins-data-public.isearchoptions.md | 2 +- ...-plugins-data-public.searchsource.fetch.md | 4 +- ...plugins-data-public.searchsource.fetch_.md | 4 +- ...ins-data-public.searchsourcefields.aggs.md | 2 +- ...-plugins-data-public.searchsourcefields.md | 2 +- ...rver.indexpatternsserviceprovider.start.md | 4 +- ...s-data-server.isearchoptions.inspector.md} | 6 +-- ...ugin-plugins-data-server.isearchoptions.md | 2 +- .../search_examples/public/search/app.tsx | 11 ++--- .../_terms_other_bucket_helper.test.ts | 5 +-- .../buckets/_terms_other_bucket_helper.ts | 4 +- .../search_source/search_source.test.ts | 45 ++++++++++++++++--- .../data/common/search/tabify/index.ts | 23 +--------- src/plugins/data/public/public.api.md | 15 ++++--- src/plugins/data/server/server.api.md | 10 +++-- 16 files changed, 77 insertions(+), 68 deletions(-) rename docs/development/plugins/data/public/{kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md => kibana-plugin-plugins-data-public.isearchoptions.inspector.md} (52%) rename docs/development/plugins/data/server/{kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md => kibana-plugin-plugins-data-server.isearchoptions.inspector.md} (52%) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md similarity index 52% rename from docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md rename to docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md index b4431b9467b71..da994498bdb39 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md) -## ISearchOptions.requestResponder property +## ISearchOptions.inspector property Signature: ```typescript -requestResponder?: RequestResponder; +inspector?: IInspectorInfo; ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md index cc0cb538be611..54a2030c68eeb 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md @@ -16,10 +16,10 @@ export interface ISearchOptions | --- | --- | --- | | [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | AbortSignal | An AbortSignal that allows the caller of search to abort a search request. | | [indexPattern](./kibana-plugin-plugins-data-public.isearchoptions.indexpattern.md) | IndexPattern | Index pattern reference is used for better error messages | +| [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md) | IInspectorInfo | | | [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | -| [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md index 623d6366d4d13..e6ba1a51a867d 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md @@ -14,7 +14,7 @@ Fetch this source and reject the returned Promise on error Signature: ```typescript -fetch(options?: ISearchOptions): Promise>; +fetch(options?: ISearchOptions): Promise>; ``` ## Parameters @@ -25,5 +25,5 @@ fetch(options?: ISearchOptions): PromiseReturns: -`Promise>` +`Promise>` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md index d5641107a88aa..4369cf7c087da 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md @@ -9,7 +9,7 @@ Fetch this source from Elasticsearch, returning an observable over the response( Signature: ```typescript -fetch$(options?: ISearchOptions): import("rxjs").Observable>; +fetch$(options?: ISearchOptions): Observable>; ``` ## Parameters @@ -20,5 +20,5 @@ fetch$(options?: ISearchOptions): import("rxjs").ObservableReturns: -`import("rxjs").Observable>` +`Observable>` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md index f6bab8e424857..12011f8242996 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md @@ -9,5 +9,5 @@ Signature: ```typescript -aggs?: any; +aggs?: object | IAggConfigs | (() => object); ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md index d0f53936eb56a..981d956a9e89b 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md @@ -16,7 +16,7 @@ export interface SearchSourceFields | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | any | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) | +| [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | object | IAggConfigs | (() => object) | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) | | [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | SearchFieldValue[] | Retrieve fields via the search Fields API | | [fieldsFromSource](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md) | NameList | Retreive fields directly from \_source (legacy behavior) | | [filter](./kibana-plugin-plugins-data-public.searchsourcefields.filter.md) | Filter[] | Filter | (() => Filter[] | Filter | undefined) | [Filter](./kibana-plugin-plugins-data-public.filter.md) | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md index 88079bb2fa3cb..118b0104fbee6 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md @@ -8,7 +8,7 @@ ```typescript start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }; ``` @@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): Returns: `{ - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md similarity index 52% rename from docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md rename to docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md index 7440f5a9d26cf..ef97b3f776874 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md) -## ISearchOptions.requestResponder property +## ISearchOptions.inspector property Signature: ```typescript -requestResponder?: RequestResponder; +inspector?: IInspectorInfo; ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md index 413a59be3d427..9b47df218a886 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md @@ -16,10 +16,10 @@ export interface ISearchOptions | --- | --- | --- | | [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | AbortSignal | An AbortSignal that allows the caller of search to abort a search request. | | [indexPattern](./kibana-plugin-plugins-data-server.isearchoptions.indexpattern.md) | IndexPattern | Index pattern reference is used for better error messages | +| [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md) | IInspectorInfo | | | [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | -| [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index 3bac445581ae7..500d7d8248cfa 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -195,13 +195,10 @@ export const SearchExamplesApp = ({ .setField('trackTotalHits', 100); if (selectedNumericField) { - searchSource.setField('aggs', () => { - return data.search.aggs - .createAggConfigs(indexPattern, [ - { type: 'avg', params: { field: selectedNumericField.name } }, - ]) - .toDsl(); - }); + const ac = data.search.aggs.createAggConfigs(indexPattern, [ + { type: 'avg', params: { field: selectedNumericField.name } }, + ]); + searchSource.setField('aggs', ac); } setRequest(searchSource.getSearchRequestBody()); diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 62cd603194250..e7661d4640cdd 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -16,7 +16,6 @@ import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; import { BUCKET_TYPES } from './bucket_agg_types'; import { IBucketAggConfig } from './bucket_agg_type'; import { mockAggTypesRegistry } from '../test_helpers'; -import { estypes } from '@elastic/elasticsearch'; const indexPattern = { id: '1234', @@ -439,7 +438,7 @@ describe('Terms Agg Other bucket helper', () => { otherAggConfig() ); - expect((mergedResponse!.aggregations!['1'].buckets as any)[1]['2'].buckets[3].key).toEqual( + expect((mergedResponse!.aggregations!['1'] as any).buckets[1]['2'].buckets[3].key).toEqual( '__other__' ); } @@ -455,7 +454,7 @@ describe('Terms Agg Other bucket helper', () => { aggConfigs.aggs[0] as IBucketAggConfig ); expect( - updatedResponse.aggregations['1'].buckets.find( + (updatedResponse!.aggregations!['1'] as any).buckets.find( (bucket: Record) => bucket.key === '__missing__' ) ).toBeDefined(); diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 576498d345f83..6230ae897b170 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -73,8 +73,8 @@ const getAggResultBuckets = ( } } } - if (responseAgg[aggWithOtherBucket.id]) { - return responseAgg[aggWithOtherBucket.id].buckets; + if (responseAgg?.[aggWithOtherBucket.id]) { + return (responseAgg[aggWithOtherBucket.id] as any).buckets; } return []; }; diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 3726e5d0c33e8..3a5a6bc3f2ef1 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -11,6 +11,8 @@ import { IndexPattern } from '../../index_patterns'; import { GetConfigFn } from '../../types'; import { fetchSoon } from './legacy'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; +import { AggConfigs } from '../../'; +import { mockAggTypesRegistry } from '../aggs/test_helpers'; jest.mock('./legacy', () => ({ fetchSoon: jest.fn().mockResolvedValue({}), @@ -39,6 +41,21 @@ const indexPattern2 = ({ getSourceFiltering: () => mockSource2, } as unknown) as IndexPattern; +const fields3 = [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }]; +const indexPattern3 = ({ + title: 'foo', + fields: { + getByName: (name: string) => { + return fields3.find((field) => field.name === name); + }, + filter: () => { + return fields3; + }, + }, + getComputedFields, + getSourceFiltering: () => mockSource, +} as unknown) as IndexPattern; + const runtimeFieldDef = { type: 'keyword', script: { @@ -81,17 +98,19 @@ describe('SearchSource', () => { describe('#getField()', () => { test('gets the value for the property', () => { - searchSource.setField('aggs', 5); - expect(searchSource.getField('aggs')).toBe(5); + searchSource.setField('aggs', { i: 5 }); + expect(searchSource.getField('aggs')).toStrictEqual({ i: 5 }); }); }); describe('#getFields()', () => { test('gets the value for the property', () => { - searchSource.setField('aggs', 5); + searchSource.setField('aggs', { i: 5 }); expect(searchSource.getFields()).toMatchInlineSnapshot(` Object { - "aggs": 5, + "aggs": Object { + "i": 5, + }, } `); }); @@ -100,7 +119,7 @@ describe('SearchSource', () => { describe('#removeField()', () => { test('remove property', () => { searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('aggs', 5); + searchSource.setField('aggs', { i: 5 }); searchSource.removeField('aggs'); expect(searchSource.getField('aggs')).toBeFalsy(); }); @@ -108,8 +127,20 @@ describe('SearchSource', () => { describe('#setField() / #flatten', () => { test('sets the value for the property', () => { - searchSource.setField('aggs', 5); - expect(searchSource.getField('aggs')).toBe(5); + searchSource.setField('aggs', { i: 5 }); + expect(searchSource.getField('aggs')).toStrictEqual({ i: 5 }); + }); + + test('sets the value for the property with AggConfigs', () => { + const typesRegistry = mockAggTypesRegistry(); + + const ac = new AggConfigs(indexPattern3, [{ type: 'avg', params: { field: 'field1' } }], { + typesRegistry, + }); + + searchSource.setField('aggs', ac); + const request = searchSource.getSearchRequestBody(); + expect(request.aggs).toStrictEqual({ '1': { avg: { field: 'field1' } } }); }); describe('computed fields handling', () => { diff --git a/src/plugins/data/common/search/tabify/index.ts b/src/plugins/data/common/search/tabify/index.ts index 168d4cf9d4c37..74fbc7ba4cfa4 100644 --- a/src/plugins/data/common/search/tabify/index.ts +++ b/src/plugins/data/common/search/tabify/index.ts @@ -6,27 +6,6 @@ * Side Public License, v 1. */ -import { SearchResponse } from 'elasticsearch'; -import { SearchSource } from '../search_source'; -import { tabifyAggResponse } from './tabify'; -import { tabifyDocs, TabifyDocsOptions } from './tabify_docs'; -import { TabbedResponseWriterOptions } from './types'; - -export const tabify = ( - searchSource: SearchSource, - esResponse: SearchResponse, - opts: Partial | TabifyDocsOptions -) => { - return !esResponse.aggregations - ? tabifyDocs(esResponse, searchSource.getField('index'), opts as TabifyDocsOptions) - : tabifyAggResponse( - searchSource.getField('aggs'), - esResponse, - opts as Partial - ); -}; - -export { tabifyDocs }; - +export { tabifyDocs } from './tabify_docs'; export { tabifyAggResponse } from './tabify'; export { tabifyGetColumns } from './get_columns'; diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 19ea754ed9a59..acfbee29f0534 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -46,6 +46,7 @@ import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_ import { History } from 'history'; import { Href } from 'history'; import { HttpSetup } from 'kibana/public'; +import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { IconType } from '@elastic/eui'; import { IncomingHttpHeaders } from 'http'; import { InjectedIntl } from '@kbn/i18n/react'; @@ -1669,13 +1670,13 @@ export type ISearchGeneric = >; + fetch$(options?: ISearchOptions): Observable>; // @deprecated - fetch(options?: ISearchOptions): Promise>; + fetch(options?: ISearchOptions): Promise>; getField(field: K, recurse?: boolean): SearchSourceFields[K]; getFields(): SearchSourceFields; getId(): string; @@ -2451,7 +2452,7 @@ export class SearchSource { // @public export interface SearchSourceFields { // (undocumented) - aggs?: any; + aggs?: object | IAggConfigs_2 | (() => object); // Warning: (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts fields?: SearchFieldValue[]; // @deprecated diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index e18db0bba16be..85b83865bd070 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -26,12 +26,14 @@ import { Ensure } from '@kbn/utility-types'; import { EnvironmentMode } from '@kbn/config'; import { ErrorToastOptions } from 'src/core/public/notifications'; import { estypes } from '@elastic/elasticsearch'; +import { EventEmitter } from 'events'; import { ExecutionContext } from 'src/plugins/expressions/common'; import { ExpressionAstExpression } from 'src/plugins/expressions/common'; import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ExpressionValueBoxed } from 'src/plugins/expressions/common'; import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_formats/utils'; +import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { ISavedObjectsRepository } from 'src/core/server'; import { IScopedClusterClient } from 'src/core/server'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; @@ -998,13 +1000,13 @@ export interface IScopedSearchClient extends ISearchClient { export interface ISearchOptions { abortSignal?: AbortSignal; indexPattern?: IndexPattern; + // Warning: (ae-forgotten-export) The symbol "IInspectorInfo" needs to be exported by the entry point index.d.ts + // + // (undocumented) + inspector?: IInspectorInfo; isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; - // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts - // - // (undocumented) - requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } From b9301dd35f9e4ee7f2e715805dab277cfad01474 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 11 Apr 2021 12:47:01 +0300 Subject: [PATCH 05/36] hierarchical param in aggconfig. ts improvements more inspector tests --- ...ins-data-public.aggconfigs.hierarchical.md | 11 ++++ ...a-plugin-plugins-data-public.aggconfigs.md | 3 +- ...in-plugins-data-public.aggconfigs.todsl.md | 9 +-- .../common/search/aggs/agg_configs.test.ts | 8 +-- .../data/common/search/aggs/agg_configs.ts | 8 ++- .../esaggs/request_handler.test.ts | 43 +++--------- .../expressions/esaggs/request_handler.ts | 3 +- .../search_source/inspect/inspector_stats.ts | 2 +- .../search_source/search_source.test.ts | 65 +++++++++++++++++++ .../search/search_source/search_source.ts | 27 +++++--- src/plugins/data/public/public.api.md | 4 +- .../data/public/search/expressions/esaggs.ts | 7 +- .../data/server/search/expressions/esaggs.ts | 8 +-- 13 files changed, 128 insertions(+), 70 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md new file mode 100644 index 0000000000000..66d540c48c3bc --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) + +## AggConfigs.hierarchical property + +Signature: + +```typescript +hierarchical?: boolean; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index 22f8994747aa2..02e9a63d95ba3 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -22,6 +22,7 @@ export declare class AggConfigs | --- | --- | --- | --- | | [aggs](./kibana-plugin-plugins-data-public.aggconfigs.aggs.md) | | IAggConfig[] | | | [createAggConfig](./kibana-plugin-plugins-data-public.aggconfigs.createaggconfig.md) | | <T extends AggConfig = AggConfig>(params: CreateAggConfigParams, { addToAggConfigs }?: {
addToAggConfigs?: boolean | undefined;
}) => T | | +| [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) | | boolean | | | [indexPattern](./kibana-plugin-plugins-data-public.aggconfigs.indexpattern.md) | | IndexPattern | | | [timeFields](./kibana-plugin-plugins-data-public.aggconfigs.timefields.md) | | string[] | | | [timeRange](./kibana-plugin-plugins-data-public.aggconfigs.timerange.md) | | TimeRange | | @@ -46,5 +47,5 @@ export declare class AggConfigs | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | -| [toDsl(hierarchical)](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | +| [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md index 055c4113ca3e4..1327e976db0ce 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md @@ -7,15 +7,8 @@ Signature: ```typescript -toDsl(hierarchical?: boolean): Record; +toDsl(): Record; ``` - -## Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| hierarchical | boolean | | - Returns: `Record` diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index 3ce528e6ed893..28102544ae055 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -342,8 +342,8 @@ describe('AggConfigs', () => { { enabled: true, type: 'max', schema: 'metric', params: { field: 'bytes' } }, ]; - const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); - const topLevelDsl = ac.toDsl(true); + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry, hierarchical: true }); + const topLevelDsl = ac.toDsl(); const buckets = ac.bySchemaName('buckets'); const metrics = ac.bySchemaName('metrics'); @@ -412,8 +412,8 @@ describe('AggConfigs', () => { }, ]; - const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); - const topLevelDsl = ac.toDsl(true)['2']; + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry, hierarchical: true }); + const topLevelDsl = ac.toDsl()['2']; expect(Object.keys(topLevelDsl.aggs)).toContain('1'); expect(Object.keys(topLevelDsl.aggs)).toContain('1-bucket'); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 4d5d49754387d..2932ef7325aed 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -43,6 +43,7 @@ function parseParentAggs(dslLvlCursor: any, dsl: any) { export interface AggConfigsOptions { typesRegistry: AggTypesRegistryStart; + hierarchical?: boolean; } export type CreateAggConfigParams = Assign; @@ -65,6 +66,8 @@ export class AggConfigs { public indexPattern: IndexPattern; public timeRange?: TimeRange; public timeFields?: string[]; + public hierarchical?: boolean = false; + private readonly typesRegistry: AggTypesRegistryStart; aggs: IAggConfig[]; @@ -80,6 +83,7 @@ export class AggConfigs { this.aggs = []; this.indexPattern = indexPattern; + this.hierarchical = opts.hierarchical; configStates.forEach((params: any) => this.createAggConfig(params)); } @@ -174,12 +178,12 @@ export class AggConfigs { return true; } - toDsl(hierarchical: boolean = false): Record { + toDsl(): Record { const dslTopLvl = {}; let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; - if (hierarchical) { + if (this.hierarchical) { // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs .filter(function (agg) { diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts index c2566535916a8..b30e5740fa3fb 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts @@ -9,7 +9,7 @@ import type { MockedKeys } from '@kbn/utility-types/jest'; import type { Filter } from '../../../es_query'; import type { IndexPattern } from '../../../index_patterns'; -import type { IAggConfig, IAggConfigs } from '../../aggs'; +import type { IAggConfigs } from '../../aggs'; import type { ISearchSource } from '../../search_source'; import { searchSourceCommonMock } from '../../search_source/mocks'; @@ -38,7 +38,6 @@ describe('esaggs expression function - public', () => { filters: undefined, indexPattern: ({ id: 'logstash-*' } as unknown) as jest.Mocked, inspectorAdapters: {}, - metricsAtAllLevels: false, partialRows: false, query: undefined, searchSessionId: 'abc123', @@ -76,21 +75,7 @@ describe('esaggs expression function - public', () => { test('setField(aggs)', async () => { expect(searchSource.setField).toHaveBeenCalledTimes(5); - expect(typeof (searchSource.setField as jest.Mock).mock.calls[2][1]).toBe('function'); - expect((searchSource.setField as jest.Mock).mock.calls[2][1]()).toEqual( - mockParams.aggs.toDsl() - ); - expect(mockParams.aggs.toDsl).toHaveBeenCalledWith(mockParams.metricsAtAllLevels); - - // make sure param is passed through - jest.clearAllMocks(); - await handleRequest({ - ...mockParams, - metricsAtAllLevels: true, - }); - searchSource = await mockParams.searchSourceService.create(); - (searchSource.setField as jest.Mock).mock.calls[2][1](); - expect(mockParams.aggs.toDsl).toHaveBeenCalledWith(true); + expect((searchSource.setField as jest.Mock).mock.calls[2][1]).toEqual(mockParams.aggs); }); test('setField(filter)', async () => { @@ -133,36 +118,24 @@ describe('esaggs expression function - public', () => { test('calls searchSource.fetch', async () => { await handleRequest(mockParams); const searchSource = await mockParams.searchSourceService.create(); + expect(searchSource.fetch$).toHaveBeenCalledWith({ abortSignal: mockParams.abortSignal, sessionId: mockParams.searchSessionId, + inspector: { + title: 'Data', + description: 'This request queries Elasticsearch to fetch the data for the visualization.', + adapter: undefined, + }, }); }); - test('calls agg.postFlightRequest if it exiests and agg is enabled', async () => { - mockParams.aggs.aggs[0].enabled = true; - await handleRequest(mockParams); - expect(mockParams.aggs.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(1); - - // ensure it works if the function doesn't exist - jest.clearAllMocks(); - mockParams.aggs.aggs[0] = ({ type: { name: 'count' } } as unknown) as IAggConfig; - expect(async () => await handleRequest(mockParams)).not.toThrowError(); - }); - - test('should skip agg.postFlightRequest call if the agg is disabled', async () => { - mockParams.aggs.aggs[0].enabled = false; - await handleRequest(mockParams); - expect(mockParams.aggs.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(0); - }); - test('tabifies response data', async () => { await handleRequest(mockParams); expect(tabifyAggResponse).toHaveBeenCalledWith( mockParams.aggs, {}, { - metricsAtAllLevels: mockParams.metricsAtAllLevels, partialRows: mockParams.partialRows, timeRange: mockParams.timeRange, } diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 2f50a8daf08b3..173b2067cad6b 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -46,7 +46,6 @@ export const handleRequest = async ({ filters, indexPattern, inspectorAdapters, - metricsAtAllLevels, partialRows, query, searchSessionId, @@ -131,7 +130,7 @@ export const handleRequest = async ({ const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null; const tabifyParams = { - metricsAtAllLevels, + metricsAtAllLevels: aggs.hierarchical, partialRows, timeRange: parsedTimeRange ? { from: parsedTimeRange.min, to: parsedTimeRange.max, timeFields: allTimeFields } diff --git a/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts index 24507a7e13058..e5a3acc23eee8 100644 --- a/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts +++ b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts @@ -50,7 +50,7 @@ export function getRequestInspectorStats(searchSource: ISearchSource) { /** @public */ export function getResponseInspectorStats( - resp: estypes.SearchResponse, + resp?: estypes.SearchResponse, searchSource?: ISearchSource ) { const lastRequest = diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 3a5a6bc3f2ef1..645eb96cc82a2 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -915,4 +915,69 @@ describe('SearchSource', () => { ); }); }); + + describe('fetch$', () => { + test('calls inspector if provided', async () => { + const requestResponder = { + stats: jest.fn(), + ok: jest.fn(), + error: jest.fn(), + json: jest.fn(), + }; + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource.fetch$(options).toPromise(); + + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.error).not.toBeCalled(); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(1); + expect(requestResponder.stats).toBeCalledTimes(2); + }); + + test('calls error on inspector', async () => { + const requestResponder = { + stats: jest.fn(), + ok: jest.fn(), + error: jest.fn(), + json: jest.fn(), + }; + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSourceDependencies.search = jest.fn().mockReturnValue(of(Promise.reject('aaaaa'))); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource + .fetch$(options) + .toPromise() + .catch(() => {}); + + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(0); + expect(requestResponder.error).toBeCalledTimes(1); + expect(requestResponder.stats).toBeCalledTimes(1); + }); + + test.skip('doesnt call any post flight requests if unavailable', () => {}); + + test.skip('calls post flight requests, fires 1 extra response', () => {}); + }); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 9eb8b42fa1fbf..584749a89c838 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -60,7 +60,7 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; -import { catchError, finalize, map, switchMap, tap } from 'rxjs/operators'; +import { catchError, finalize, last, map, share, switchMap, tap } from 'rxjs/operators'; import { defer, from, Observable } from 'rxjs'; import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; @@ -271,7 +271,8 @@ export class SearchSource { description, searchSessionId: options.sessionId, }); - return defer(() => this.requestIsStarting(options)).pipe( + + const s$ = defer(() => this.requestIsStarting(options)).pipe( tap(() => { requestResponder?.stats(getRequestInspectorStats(this)); }), @@ -288,11 +289,8 @@ export class SearchSource { }), tap((response) => { // TODO: Remove casting when https://github.com/elastic/elasticsearch-js/issues/1287 is resolved - if ((response as any).error) { + if (!response || (response as any).error) { throw new RequestFailure(null, response); - } else { - requestResponder?.stats(getResponseInspectorStats(response, this)); - requestResponder?.ok({ json: response }); } }), catchError((e) => { @@ -301,8 +299,19 @@ export class SearchSource { }), finalize(() => { requestResponder?.json(this.getSearchRequestBody()); - }) + }), + share() ); + + const sub = s$.pipe(last()).subscribe({ + next: (finalResponse) => { + requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); + requestResponder?.ok({ json: finalResponse }); + sub.unsubscribe(); + }, + }); + + return s$; } /** @@ -393,7 +402,9 @@ export class SearchSource { } else { if (!this.hasPostFlightRequests()) { obs.next(response); - obs.complete(); + if (isCompleteResponse(response)) { + obs.complete(); + } } else { obs.next({ ...response, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index ee87086ee0ba8..35ea1cfedfc95 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -255,6 +255,8 @@ export class AggConfigs { getResponseAggById(id: string): AggConfig | undefined; getResponseAggs(): AggConfig[]; // (undocumented) + hierarchical?: boolean; + // (undocumented) indexPattern: IndexPattern; jsonDataEquals(aggConfigs: AggConfig[]): boolean; // (undocumented) @@ -268,7 +270,7 @@ export class AggConfigs { // (undocumented) timeRange?: TimeRange; // (undocumented) - toDsl(hierarchical?: boolean): Record; + toDsl(): Record; } // @internal (undocumented) diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index 45d24af3a6ebb..1e3d56c71e423 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -8,7 +8,6 @@ import { get } from 'lodash'; import { StartServicesAccessor } from 'src/core/public'; -import { Adapters } from 'src/plugins/inspector/common'; import { EsaggsExpressionFunctionDefinition, EsaggsStartDependencies, @@ -44,14 +43,14 @@ export function getFunctionDefinition({ indexPattern, args.aggs!.map((agg) => agg.value) ); + aggConfigs.hierarchical = args.metricsAtAllLevels; return await handleEsaggsRequest({ - abortSignal: (abortSignal as unknown) as AbortSignal, + abortSignal, aggs: aggConfigs, filters: get(input, 'filters', undefined), indexPattern, - inspectorAdapters: inspectorAdapters as Adapters, - metricsAtAllLevels: args.metricsAtAllLevels, + inspectorAdapters, partialRows: args.partialRows, query: get(input, 'query', undefined) as any, searchSessionId: getSearchSessionId(), diff --git a/src/plugins/data/server/search/expressions/esaggs.ts b/src/plugins/data/server/search/expressions/esaggs.ts index 61fd320d89b95..bb22a491b157e 100644 --- a/src/plugins/data/server/search/expressions/esaggs.ts +++ b/src/plugins/data/server/search/expressions/esaggs.ts @@ -9,7 +9,6 @@ import { get } from 'lodash'; import { i18n } from '@kbn/i18n'; import { KibanaRequest, StartServicesAccessor } from 'src/core/server'; -import { Adapters } from 'src/plugins/inspector/common'; import { EsaggsExpressionFunctionDefinition, EsaggsStartDependencies, @@ -61,13 +60,14 @@ export function getFunctionDefinition({ args.aggs!.map((agg) => agg.value) ); + aggConfigs.hierarchical = args.metricsAtAllLevels; + return await handleEsaggsRequest({ - abortSignal: (abortSignal as unknown) as AbortSignal, + abortSignal, aggs: aggConfigs, filters: get(input, 'filters', undefined), indexPattern, - inspectorAdapters: inspectorAdapters as Adapters, - metricsAtAllLevels: args.metricsAtAllLevels, + inspectorAdapters, partialRows: args.partialRows, query: get(input, 'query', undefined) as any, searchSessionId: getSearchSessionId(), From e88df9d0920c3556f9cf8b95ae41234f7a043c51 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 11 Apr 2021 16:23:35 +0300 Subject: [PATCH 06/36] fix jest --- src/plugins/data/server/search/expressions/esaggs.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/server/search/expressions/esaggs.test.ts b/src/plugins/data/server/search/expressions/esaggs.test.ts index 124a171de6378..15287e9d8cf5b 100644 --- a/src/plugins/data/server/search/expressions/esaggs.test.ts +++ b/src/plugins/data/server/search/expressions/esaggs.test.ts @@ -108,11 +108,13 @@ describe('esaggs expression function - server', () => { expect(handleEsaggsRequest).toHaveBeenCalledWith({ abortSignal: mockHandlers.abortSignal, - aggs: { foo: 'bar' }, + aggs: { + foo: 'bar', + hierarchical: args.metricsAtAllLevels, + }, filters: undefined, indexPattern: {}, inspectorAdapters: mockHandlers.inspectorAdapters, - metricsAtAllLevels: args.metricsAtAllLevels, partialRows: args.partialRows, query: undefined, searchSessionId: 'abc123', From 47b994438aca5fe2746bf0fe6c5304aa3528ee30 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 11 Apr 2021 23:48:31 +0300 Subject: [PATCH 07/36] separate inspect more tests --- .../search_source/search_source.test.ts | 69 +++++++++++-- .../search/search_source/search_source.ts | 96 +++++++++++++------ 2 files changed, 126 insertions(+), 39 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 645eb96cc82a2..8b4728c50282e 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -78,8 +78,8 @@ describe('SearchSource', () => { .fn() .mockReturnValue( of( - { rawResponse: { isPartial: true, isRunning: true } }, - { rawResponse: { isPartial: false, isRunning: false } } + { rawResponse: { test: 1 }, isPartial: true, isRunning: true }, + { rawResponse: { test: 2 }, isPartial: false, isRunning: false } ) ); @@ -739,16 +739,14 @@ describe('SearchSource', () => { expect(next.mock.calls[0]).toMatchInlineSnapshot(` Array [ Object { - "isPartial": true, - "isRunning": true, + "test": 1, }, ] `); expect(next.mock.calls[1]).toMatchInlineSnapshot(` Array [ Object { - "isPartial": false, - "isRunning": false, + "test": 2, }, ] `); @@ -941,6 +939,7 @@ describe('SearchSource', () => { expect(requestResponder.error).not.toBeCalled(); expect(requestResponder.json).toBeCalledTimes(1); expect(requestResponder.ok).toBeCalledTimes(1); + // First and last expect(requestResponder.stats).toBeCalledTimes(2); }); @@ -971,13 +970,63 @@ describe('SearchSource', () => { expect(options.inspector.adapter.start).toBeCalledTimes(1); expect(requestResponder.json).toBeCalledTimes(1); - expect(requestResponder.ok).toBeCalledTimes(0); expect(requestResponder.error).toBeCalledTimes(1); - expect(requestResponder.stats).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(0); + expect(requestResponder.stats).toBeCalledTimes(0); + }); + + test('doesnt call any post flight requests if disabled', async () => { + const typesRegistry = mockAggTypesRegistry(); + + const ac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: false, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + + ac.aggs[0].type.postFlightRequest = jest.fn(); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + await searchSource.fetch$({}).toPromise(); + + expect(ac.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(0); }); - test.skip('doesnt call any post flight requests if unavailable', () => {}); + test.skip('calls post flight requests, fires 1 extra response', async () => { + const typesRegistry = mockAggTypesRegistry(); + + const ac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: true, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + + ac.aggs[0].type.postFlightRequest = jest.fn(); - test.skip('calls post flight requests, fires 1 extra response', () => {}); + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + await searchSource.fetch$({}).toPromise(); + + expect(ac.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 584749a89c838..55cc97b4a833e 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -60,8 +60,17 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; -import { catchError, finalize, last, map, share, switchMap, tap } from 'rxjs/operators'; -import { defer, from, Observable } from 'rxjs'; +import { + catchError, + finalize, + first, + last, + map, + shareReplay, + switchMap, + tap, +} from 'rxjs/operators'; +import { defer, EMPTY, from, Observable } from 'rxjs'; import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; @@ -265,17 +274,7 @@ export class SearchSource { fetch$(options: ISearchOptions = {}) { const { getConfig } = this.dependencies; - const { id, title, description, adapter } = options.inspector || { title: '' }; - const requestResponder = adapter?.start(title, { - id, - description, - searchSessionId: options.sessionId, - }); - const s$ = defer(() => this.requestIsStarting(options)).pipe( - tap(() => { - requestResponder?.stats(getRequestInspectorStats(this)); - }), switchMap(() => { const searchRequest = this.flatten(); this.history = [searchRequest]; @@ -293,25 +292,10 @@ export class SearchSource { throw new RequestFailure(null, response); } }), - catchError((e) => { - requestResponder?.error({ json: e }); - throw e; - }), - finalize(() => { - requestResponder?.json(this.getSearchRequestBody()); - }), - share() + shareReplay() ); - const sub = s$.pipe(last()).subscribe({ - next: (finalResponse) => { - requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); - requestResponder?.ok({ json: finalResponse }); - sub.unsubscribe(); - }, - }); - - return s$; + return this.inspectSearch(s$, options); } /** @@ -352,6 +336,60 @@ export class SearchSource { * PRIVATE APIS ******/ + private inspectSearch(s$: Observable>, options: ISearchOptions) { + const { id, title, description, adapter } = options.inspector || { title: '' }; + const requestResponder = adapter?.start(title, { + id, + description, + searchSessionId: options.sessionId, + }); + + try { + requestResponder?.json(this.getSearchRequestBody()); + } catch (e) { + // ignore + } + + // Track request stats on first emit + const first$ = s$ + .pipe( + first(undefined, null), + tap(() => { + requestResponder?.stats(getRequestInspectorStats(this)); + }), + catchError((e) => { + return EMPTY; + }), + finalize(() => { + first$.unsubscribe(); + }) + ) + .subscribe(); + + // Track response stats on last emit + // Also track errors + const last$ = s$ + .pipe( + catchError((e) => { + requestResponder?.error({ json: e }); + return EMPTY; + }), + last(undefined, null), + tap((finalResponse) => { + if (finalResponse) { + requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); + requestResponder?.ok({ json: finalResponse }); + } + }), + finalize(() => { + last$.unsubscribe(); + }) + ) + .subscribe(); + + return s$; + } + private hasPostFlightRequests() { const aggs = this.getField('aggs'); if (aggs instanceof AggConfigs) { From b9ce734702d729163ab391029b9c84416077553e Mon Sep 17 00:00:00 2001 From: Liza K Date: Mon, 12 Apr 2021 11:10:27 +0300 Subject: [PATCH 08/36] jest --- src/plugins/data/public/search/expressions/esaggs.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/search/expressions/esaggs.test.ts b/src/plugins/data/public/search/expressions/esaggs.test.ts index d7a6446781c43..e75bd7be219de 100644 --- a/src/plugins/data/public/search/expressions/esaggs.test.ts +++ b/src/plugins/data/public/search/expressions/esaggs.test.ts @@ -100,17 +100,20 @@ describe('esaggs expression function - public', () => { expect(handleEsaggsRequest).toHaveBeenCalledWith({ abortSignal: mockHandlers.abortSignal, - aggs: { foo: 'bar' }, + aggs: { + foo: 'bar', + hierarchical: true, + }, filters: undefined, indexPattern: {}, inspectorAdapters: mockHandlers.inspectorAdapters, - metricsAtAllLevels: args.metricsAtAllLevels, partialRows: args.partialRows, query: undefined, searchSessionId: 'abc123', searchSourceService: startDependencies.searchSource, timeFields: args.timeFields, timeRange: undefined, + getNow: undefined, }); }); From 2976277bf3b9f40f109eb69f9aeaeb19d0e60007 Mon Sep 17 00:00:00 2001 From: Liza K Date: Mon, 12 Apr 2021 11:54:10 +0300 Subject: [PATCH 09/36] inspector --- .../data/common/search/search_source/search_source.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 55cc97b4a833e..0ca6bbf8c2107 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -344,12 +344,6 @@ export class SearchSource { searchSessionId: options.sessionId, }); - try { - requestResponder?.json(this.getSearchRequestBody()); - } catch (e) { - // ignore - } - // Track request stats on first emit const first$ = s$ .pipe( @@ -376,6 +370,11 @@ export class SearchSource { }), last(undefined, null), tap((finalResponse) => { + try { + requestResponder?.json(this.getSearchRequestBody()); + } catch (e) { + // ignore + } if (finalResponse) { requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); requestResponder?.ok({ json: finalResponse }); From 8d4aafe16c63f728653718b37216416db3d9e321 Mon Sep 17 00:00:00 2001 From: Liza K Date: Mon, 12 Apr 2021 15:54:01 +0300 Subject: [PATCH 10/36] Error handling and more tests --- .../search_source/search_source.test.ts | 503 ++++++++++++------ .../search/search_source/search_source.ts | 36 +- 2 files changed, 376 insertions(+), 163 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 8b4728c50282e..7f8a4fceff05d 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -11,8 +11,10 @@ import { IndexPattern } from '../../index_patterns'; import { GetConfigFn } from '../../types'; import { fetchSoon } from './legacy'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; -import { AggConfigs } from '../../'; +import { AggConfigs, AggTypesRegistryStart } from '../../'; import { mockAggTypesRegistry } from '../aggs/test_helpers'; +import { RequestResponder } from 'src/plugins/inspector/common'; +import { switchMap } from 'rxjs/operators'; jest.mock('./legacy', () => ({ fetchSoon: jest.fn().mockResolvedValue({}), @@ -662,7 +664,7 @@ describe('SearchSource', () => { const fn = jest.fn(); searchSource.onRequestStart(fn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); }); @@ -675,7 +677,7 @@ describe('SearchSource', () => { const parentFn = jest.fn(); parent.onRequestStart(parentFn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); expect(parentFn).not.toBeCalled(); @@ -695,67 +697,13 @@ describe('SearchSource', () => { const parentFn = jest.fn(); parent.onRequestStart(parentFn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); expect(parentFn).toBeCalledWith(searchSource, options); }); }); - describe('#legacy fetch()', () => { - beforeEach(() => { - searchSourceDependencies = { - ...searchSourceDependencies, - getConfig: jest.fn(() => { - return true; // batchSearches = true - }) as GetConfigFn, - }; - }); - - test('should call msearch', async () => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - await searchSource.fetch(options); - expect(fetchSoon).toBeCalledTimes(1); - }); - }); - - describe('#search service fetch()', () => { - test('should call msearch', async () => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - - await searchSource.fetch(options); - expect(mockSearchMethod).toBeCalledTimes(1); - }); - - test('should return partial results', (done) => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - - const next = jest.fn(); - const complete = () => { - expect(next).toBeCalledTimes(2); - expect(next.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Object { - "test": 1, - }, - ] - `); - expect(next.mock.calls[1]).toMatchInlineSnapshot(` - Array [ - Object { - "test": 2, - }, - ] - `); - done(); - }; - searchSource.fetch$(options).subscribe({ next, complete }); - }); - }); - describe('#serialize', () => { test('should reference index patterns', () => { const indexPattern123 = { id: '123' } as IndexPattern; @@ -915,118 +863,371 @@ describe('SearchSource', () => { }); describe('fetch$', () => { - test('calls inspector if provided', async () => { - const requestResponder = { - stats: jest.fn(), - ok: jest.fn(), - error: jest.fn(), - json: jest.fn(), - }; - const options = { - inspector: { - title: 'a', - adapter: { - start: jest.fn().mockReturnValue(requestResponder), - } as any, - }, - }; - - searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('index', indexPattern); - await searchSource.fetch$(options).toPromise(); + describe('#legacy fetch()', () => { + beforeEach(() => { + searchSourceDependencies = { + ...searchSourceDependencies, + getConfig: jest.fn(() => { + return true; // batchSearches = true + }) as GetConfigFn, + }; + }); - expect(options.inspector.adapter.start).toBeCalledTimes(1); - expect(requestResponder.error).not.toBeCalled(); - expect(requestResponder.json).toBeCalledTimes(1); - expect(requestResponder.ok).toBeCalledTimes(1); - // First and last - expect(requestResponder.stats).toBeCalledTimes(2); + test('should call msearch', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + await searchSource.fetch$(options).toPromise(); + expect(fetchSoon).toBeCalledTimes(1); + }); }); - test('calls error on inspector', async () => { - const requestResponder = { - stats: jest.fn(), - ok: jest.fn(), - error: jest.fn(), - json: jest.fn(), - }; - const options = { - inspector: { - title: 'a', - adapter: { - start: jest.fn().mockReturnValue(requestResponder), - } as any, - }, - }; + describe('responses', () => { + test('should return partial results', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; - searchSourceDependencies.search = jest.fn().mockReturnValue(of(Promise.reject('aaaaa'))); + const next = jest.fn(); + const complete = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, complete }); + await res$.toPromise(); - searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('index', indexPattern); - await searchSource - .fetch$(options) - .toPromise() - .catch(() => {}); - - expect(options.inspector.adapter.start).toBeCalledTimes(1); - expect(requestResponder.json).toBeCalledTimes(1); - expect(requestResponder.error).toBeCalledTimes(1); - expect(requestResponder.ok).toBeCalledTimes(0); - expect(requestResponder.stats).toBeCalledTimes(0); + expect(next).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(1); + expect(next.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 1, + }, + ] + `); + expect(next.mock.calls[1]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 2, + }, + ] + `); + }); + + test('shareReplays result', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + + const next = jest.fn(); + const complete = jest.fn(); + const next2 = jest.fn(); + const complete2 = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, complete }); + res$.subscribe({ next: next2, complete: complete2 }); + await res$.toPromise(); + + expect(next).toBeCalledTimes(2); + expect(next2).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(1); + expect(complete2).toBeCalledTimes(1); + expect(searchSourceDependencies.search).toHaveBeenCalledTimes(1); + }); + + test('should emit error on empty response', async () => { + searchSourceDependencies.search = mockSearchMethod = jest + .fn() + .mockReturnValue( + of({ rawResponse: { test: 1 }, isPartial: true, isRunning: true }, undefined) + ); + + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + + const next = jest.fn(); + const error = jest.fn(); + const complete = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, error, complete }); + await res$.toPromise().catch((e) => {}); + + expect(next).toBeCalledTimes(1); + expect(error).toBeCalledTimes(1); + expect(complete).toBeCalledTimes(0); + expect(next.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 1, + }, + ] + `); + expect(error.mock.calls[0][0]).toBe(undefined); + }); }); - test('doesnt call any post flight requests if disabled', async () => { - const typesRegistry = mockAggTypesRegistry(); + describe('inspector', () => { + let requestResponder: RequestResponder; + beforeEach(() => { + requestResponder = ({ + stats: jest.fn(), + ok: jest.fn(), + error: jest.fn(), + json: jest.fn(), + } as unknown) as RequestResponder; + }); - const ac = new AggConfigs( - indexPattern3, - [ - { - type: 'avg', - enabled: false, - params: { field: 'field1' }, + test('calls inspector if provided', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, }, - ], - { - typesRegistry, - } - ); + }; - ac.aggs[0].type.postFlightRequest = jest.fn(); + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource.fetch$(options).toPromise(); - searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('index', indexPattern); - searchSource.setField('aggs', ac); - await searchSource.fetch$({}).toPromise(); + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.error).not.toBeCalled(); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(1); + // First and last + expect(requestResponder.stats).toBeCalledTimes(2); + }); + + test('calls inspector only once, with multiple subs (shareReplay)', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; - expect(ac.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(0); + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + const res$ = searchSource.fetch$(options); + + const complete1 = jest.fn(); + const complete2 = jest.fn(); + + res$.subscribe({ + complete: complete1, + }); + res$.subscribe({ + complete: complete2, + }); + + await res$.toPromise(); + + expect(complete1).toBeCalledTimes(1); + expect(complete2).toBeCalledTimes(1); + expect(options.inspector.adapter.start).toBeCalledTimes(1); + }); + + test('calls error on inspector', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSourceDependencies.search = jest.fn().mockReturnValue(of(Promise.reject('aaaaa'))); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource + .fetch$(options) + .toPromise() + .catch(() => {}); + + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.error).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(0); + expect(requestResponder.stats).toBeCalledTimes(0); + }); }); - test.skip('calls post flight requests, fires 1 extra response', async () => { - const typesRegistry = mockAggTypesRegistry(); + describe('postFlightRequest', () => { + let fetchSub: any; + + function getAggConfigs(typesRegistry: AggTypesRegistryStart, enabled: boolean) { + return new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + } + + beforeEach(() => { + fetchSub = { + next: jest.fn(), + complete: jest.fn(), + error: jest.fn(), + }; + }); + + test('doesnt call any post flight requests if disabled', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn(); + const ac = getAggConfigs(typesRegistry, false); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(2); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(fetchSub.error).toHaveBeenCalledTimes(0); + + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(0); + }); + + test('doesnt call any post flight if searchsource has error', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn(); + const ac = getAggConfigs(typesRegistry, true); + + searchSourceDependencies.search = jest.fn().mockImplementation(() => + of(1).pipe( + switchMap((r) => { + throw r; + }) + ) + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + await fetch$.toPromise().catch((e) => {}); + + expect(fetchSub.next).toHaveBeenCalledTimes(0); + expect(fetchSub.complete).toHaveBeenCalledTimes(0); + expect(fetchSub.error).toHaveBeenNthCalledWith(1, 1); + + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(0); + }); + + test('calls post flight requests, fires 1 extra response, returns last response', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockResolvedValue({ + other: 5, + }); - const ac = new AggConfigs( - indexPattern3, - [ + const allac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: true, + params: { field: 'field1' }, + }, + { + type: 'avg', + enabled: true, + params: { field: 'field2' }, + }, + { + type: 'avg', + enabled: true, + params: { field: 'foo-bar' }, + }, + ], { - type: 'avg', - enabled: true, - params: { field: 'field1' }, - }, - ], - { - typesRegistry, - } - ); + typesRegistry, + } + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', allac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + const resp = await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(3); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(fetchSub.error).toHaveBeenCalledTimes(0); + expect(resp).toStrictEqual({ other: 5 }); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(3); + }); - ac.aggs[0].type.postFlightRequest = jest.fn(); + test('calls post flight requests only once, with multiple subs (shareReplay)', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockResolvedValue({ + other: 5, + }); - searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('index', indexPattern); - searchSource.setField('aggs', ac); - await searchSource.fetch$({}).toPromise(); + const allac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: true, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', allac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + const fetchSub2 = { + next: jest.fn(), + complete: jest.fn(), + error: jest.fn(), + }; + fetch$.subscribe(fetchSub2); + + await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(3); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(1); + }); - expect(ac.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(1); + test('calls post flight requests, handles error', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockRejectedValue(undefined); + const ac = getAggConfigs(typesRegistry, true); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + await fetch$.toPromise().catch(() => {}); + + expect(fetchSub.next).toHaveBeenCalledTimes(2); + expect(fetchSub.complete).toHaveBeenCalledTimes(0); + expect(fetchSub.error).toHaveBeenCalledTimes(1); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(1); + }); }); }); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 0ca6bbf8c2107..a47ad13683051 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -91,6 +91,8 @@ import { Filter, UI_SETTINGS, isCompleteResponse, + isErrorResponse, + isPartialResponse, IKibanaSearchResponse, } from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; @@ -338,20 +340,29 @@ export class SearchSource { private inspectSearch(s$: Observable>, options: ISearchOptions) { const { id, title, description, adapter } = options.inspector || { title: '' }; + const requestResponder = adapter?.start(title, { id, description, searchSessionId: options.sessionId, }); - // Track request stats on first emit + const trackRequestBody = () => { + try { + requestResponder?.json(this.getSearchRequestBody()); + } catch (e) {} // eslint-disable-line no-empty + }; + + // Track request stats on first emit, swallow errors const first$ = s$ .pipe( first(undefined, null), tap(() => { + trackRequestBody(); requestResponder?.stats(getRequestInspectorStats(this)); }), - catchError((e) => { + catchError(() => { + trackRequestBody(); return EMPTY; }), finalize(() => { @@ -360,8 +371,7 @@ export class SearchSource { ) .subscribe(); - // Track response stats on last emit - // Also track errors + // Track response stats on last emit, as well as errors const last$ = s$ .pipe( catchError((e) => { @@ -370,11 +380,6 @@ export class SearchSource { }), last(undefined, null), tap((finalResponse) => { - try { - requestResponder?.json(this.getSearchRequestBody()); - } catch (e) { - // ignore - } if (finalResponse) { requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); requestResponder?.ok({ json: finalResponse }); @@ -422,7 +427,7 @@ export class SearchSource { /** * Run a search using the search service - * @return {Promise>} + * @return {Observable>} */ private fetchSearch$(searchRequest: SearchRequest, options: ISearchOptions) { const { search, getConfig, onResponse } = this.dependencies; @@ -434,7 +439,9 @@ export class SearchSource { return search({ params, indexType: searchRequest.indexType }, options).pipe( switchMap((response) => { return new Observable>((obs) => { - if (!isCompleteResponse(response)) { + if (isErrorResponse(response)) { + obs.error(response); + } else if (isPartialResponse(response)) { obs.next(response); } else { if (!this.hasPostFlightRequests()) { @@ -443,6 +450,7 @@ export class SearchSource { obs.complete(); } } else { + // Treat the complete response as partial, then run the postFlightRequests. obs.next({ ...response, isPartial: true, @@ -455,9 +463,13 @@ export class SearchSource { rawResponse: responseWithOther, }); }, - complete: () => { + error: (e) => { + obs.error(e); sub.unsubscribe(); + }, + complete: () => { obs.complete(); + sub.unsubscribe(); }, }); } From c7514eee6f033c5d90b0ff5950ad3ae8f8e0378b Mon Sep 17 00:00:00 2001 From: Liza K Date: Tue, 13 Apr 2021 11:09:47 +0300 Subject: [PATCH 11/36] put the fun in functional tests --- src/plugins/data/common/search/search_source/search_source.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index a47ad13683051..d7f7973ae936c 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -358,8 +358,8 @@ export class SearchSource { .pipe( first(undefined, null), tap(() => { - trackRequestBody(); requestResponder?.stats(getRequestInspectorStats(this)); + trackRequestBody(); }), catchError(() => { trackRequestBody(); From 500c226052a57cbd2c9c750f8b142a3b128c83fb Mon Sep 17 00:00:00 2001 From: Liza K Date: Tue, 13 Apr 2021 13:03:29 +0300 Subject: [PATCH 12/36] delete client side legacy msearch code --- .../search_source/legacy/call_client.test.ts | 102 -------------- .../search_source/legacy/call_client.ts | 38 ----- .../search_source/legacy/fetch_soon.test.ts | 132 ------------------ .../search/search_source/legacy/fetch_soon.ts | 83 ----------- .../search/search_source/legacy/index.ts | 1 - .../search_source/search_source.test.ts | 23 --- .../search/search_source/search_source.ts | 28 +--- .../data/public/search/search_service.ts | 11 +- 8 files changed, 10 insertions(+), 408 deletions(-) delete mode 100644 src/plugins/data/common/search/search_source/legacy/call_client.test.ts delete mode 100644 src/plugins/data/common/search/search_source/legacy/call_client.ts delete mode 100644 src/plugins/data/common/search/search_source/legacy/fetch_soon.test.ts delete mode 100644 src/plugins/data/common/search/search_source/legacy/fetch_soon.ts diff --git a/src/plugins/data/common/search/search_source/legacy/call_client.test.ts b/src/plugins/data/common/search/search_source/legacy/call_client.test.ts deleted file mode 100644 index 93849b63939e4..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/call_client.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { callClient } from './call_client'; -import { SearchStrategySearchParams } from './types'; -import { defaultSearchStrategy } from './default_search_strategy'; -import { FetchHandlers } from '../fetch'; -import { BehaviorSubject } from 'rxjs'; - -const mockAbortFn = jest.fn(); - -jest.mock('./default_search_strategy', () => { - return { - defaultSearchStrategy: { - search: jest.fn(({ searchRequests }: SearchStrategySearchParams) => { - return { - searching: Promise.resolve( - searchRequests.map((req) => { - return { - id: req._searchStrategyId, - }; - }) - ), - abort: mockAbortFn, - }; - }), - }, - }; -}); - -describe('callClient', () => { - const handleResponse = jest.fn().mockImplementation((req, res) => res); - const handlers = { - getConfig: jest.fn(), - onResponse: handleResponse, - legacy: { - callMsearch: jest.fn(), - loadingCount$: new BehaviorSubject(0), - }, - } as FetchHandlers; - - beforeEach(() => { - handleResponse.mockClear(); - }); - - test('Passes the additional arguments it is given to the search strategy', () => { - const searchRequests = [{ _searchStrategyId: 0 }]; - - callClient(searchRequests, [], handlers); - - expect(defaultSearchStrategy.search).toBeCalled(); - expect((defaultSearchStrategy.search as any).mock.calls[0][0]).toEqual({ - searchRequests, - ...handlers, - }); - }); - - test('Returns the responses in the original order', async () => { - const searchRequests = [{ _searchStrategyId: 1 }, { _searchStrategyId: 0 }]; - - const responses = await Promise.all(callClient(searchRequests, [], handlers)); - - expect(responses[0]).toEqual({ id: searchRequests[0]._searchStrategyId }); - expect(responses[1]).toEqual({ id: searchRequests[1]._searchStrategyId }); - }); - - test('Calls handleResponse with each request and response', async () => { - const searchRequests = [{ _searchStrategyId: 0 }, { _searchStrategyId: 1 }]; - - const responses = callClient(searchRequests, [], handlers); - await Promise.all(responses); - - expect(handleResponse).toBeCalledTimes(2); - expect(handleResponse).toBeCalledWith(searchRequests[0], { - id: searchRequests[0]._searchStrategyId, - }); - expect(handleResponse).toBeCalledWith(searchRequests[1], { - id: searchRequests[1]._searchStrategyId, - }); - }); - - test('If passed an abortSignal, calls abort on the strategy if the signal is aborted', () => { - const searchRequests = [{ _searchStrategyId: 0 }, { _searchStrategyId: 1 }]; - const abortController = new AbortController(); - const requestOptions = [ - { - abortSignal: abortController.signal, - }, - ]; - - callClient(searchRequests, requestOptions, handlers); - abortController.abort(); - - expect(mockAbortFn).toBeCalled(); - // expect(mockAbortFns[1]).not.toBeCalled(); - }); -}); diff --git a/src/plugins/data/common/search/search_source/legacy/call_client.ts b/src/plugins/data/common/search/search_source/legacy/call_client.ts deleted file mode 100644 index 4c1156aac7015..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/call_client.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { estypes } from '@elastic/elasticsearch'; -import { FetchHandlers, SearchRequest } from '../fetch'; -import { defaultSearchStrategy } from './default_search_strategy'; -import { ISearchOptions } from '../../index'; - -export function callClient( - searchRequests: SearchRequest[], - requestsOptions: ISearchOptions[] = [], - fetchHandlers: FetchHandlers -) { - // Correlate the options with the request that they're associated with - const requestOptionEntries: Array< - [SearchRequest, ISearchOptions] - > = searchRequests.map((request, i) => [request, requestsOptions[i]]); - const requestOptionsMap = new Map(requestOptionEntries); - const requestResponseMap = new Map>>(); - - const { searching, abort } = defaultSearchStrategy.search({ - searchRequests, - ...fetchHandlers, - }); - - searchRequests.forEach((request, i) => { - const response = searching.then((results) => fetchHandlers.onResponse(request, results[i])); - const { abortSignal = null } = requestOptionsMap.get(request) || {}; - if (abortSignal) abortSignal.addEventListener('abort', abort); - requestResponseMap.set(request, response); - }); - return searchRequests.map((request) => requestResponseMap.get(request)!); -} diff --git a/src/plugins/data/common/search/search_source/legacy/fetch_soon.test.ts b/src/plugins/data/common/search/search_source/legacy/fetch_soon.test.ts deleted file mode 100644 index eca6e75fc69de..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/fetch_soon.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { SearchResponse } from 'elasticsearch'; -import { UI_SETTINGS } from '../../../constants'; -import { GetConfigFn } from '../../../types'; -import { FetchHandlers, SearchRequest } from '../fetch'; -import { ISearchOptions } from '../../index'; -import { callClient } from './call_client'; -import { fetchSoon } from './fetch_soon'; - -function getConfigStub(config: any = {}): GetConfigFn { - return (key) => config[key]; -} - -const mockResponses: Record> = { - foo: { - took: 1, - timed_out: false, - } as SearchResponse, - bar: { - took: 2, - timed_out: false, - } as SearchResponse, - baz: { - took: 3, - timed_out: false, - } as SearchResponse, -}; - -jest.useFakeTimers(); - -jest.mock('./call_client', () => ({ - callClient: jest.fn((requests: SearchRequest[]) => { - // Allow a request object to specify which mockResponse it wants to receive (_mockResponseId) - // in addition to how long to simulate waiting before returning a response (_waitMs) - const responses = requests.map((request) => { - const waitMs = requests.reduce((total, { _waitMs }) => total + _waitMs || 0, 0); - return new Promise((resolve) => { - setTimeout(() => { - resolve(mockResponses[request._mockResponseId]); - }, waitMs); - }); - }); - return Promise.resolve(responses); - }), -})); - -describe('fetchSoon', () => { - beforeEach(() => { - (callClient as jest.Mock).mockClear(); - }); - - test('should execute asap if config is set to not batch searches', () => { - const getConfig = getConfigStub({ [UI_SETTINGS.COURIER_BATCH_SEARCHES]: false }); - const request = {}; - const options = {}; - - fetchSoon(request, options, { getConfig } as FetchHandlers); - - expect(callClient).toBeCalled(); - }); - - test('should delay by 50ms if config is set to batch searches', () => { - const getConfig = getConfigStub({ [UI_SETTINGS.COURIER_BATCH_SEARCHES]: true }); - const request = {}; - const options = {}; - - fetchSoon(request, options, { getConfig } as FetchHandlers); - - expect(callClient).not.toBeCalled(); - jest.advanceTimersByTime(0); - expect(callClient).not.toBeCalled(); - jest.advanceTimersByTime(50); - expect(callClient).toBeCalled(); - }); - - test('should send a batch of requests to callClient', () => { - const getConfig = getConfigStub({ [UI_SETTINGS.COURIER_BATCH_SEARCHES]: true }); - const requests = [{ foo: 1 }, { foo: 2 }]; - const options = [{ bar: 1 }, { bar: 2 }]; - - requests.forEach((request, i) => { - fetchSoon(request, options[i] as ISearchOptions, { getConfig } as FetchHandlers); - }); - - jest.advanceTimersByTime(50); - expect(callClient).toBeCalledTimes(1); - expect((callClient as jest.Mock).mock.calls[0][0]).toEqual(requests); - expect((callClient as jest.Mock).mock.calls[0][1]).toEqual(options); - }); - - test('should return the response to the corresponding call for multiple batched requests', async () => { - const getConfig = getConfigStub({ [UI_SETTINGS.COURIER_BATCH_SEARCHES]: true }); - const requests = [{ _mockResponseId: 'foo' }, { _mockResponseId: 'bar' }]; - - const promises = requests.map((request) => { - return fetchSoon(request, {}, { getConfig } as FetchHandlers); - }); - jest.advanceTimersByTime(50); - const results = await Promise.all(promises); - - expect(results).toEqual([mockResponses.foo, mockResponses.bar]); - }); - - test('should wait for the previous batch to start before starting a new batch', () => { - const getConfig = getConfigStub({ [UI_SETTINGS.COURIER_BATCH_SEARCHES]: true }); - const firstBatch = [{ foo: 1 }, { foo: 2 }]; - const secondBatch = [{ bar: 1 }, { bar: 2 }]; - - firstBatch.forEach((request) => { - fetchSoon(request, {}, { getConfig } as FetchHandlers); - }); - jest.advanceTimersByTime(50); - secondBatch.forEach((request) => { - fetchSoon(request, {}, { getConfig } as FetchHandlers); - }); - - expect(callClient).toBeCalledTimes(1); - expect((callClient as jest.Mock).mock.calls[0][0]).toEqual(firstBatch); - - jest.advanceTimersByTime(50); - - expect(callClient).toBeCalledTimes(2); - expect((callClient as jest.Mock).mock.calls[1][0]).toEqual(secondBatch); - }); -}); diff --git a/src/plugins/data/common/search/search_source/legacy/fetch_soon.ts b/src/plugins/data/common/search/search_source/legacy/fetch_soon.ts deleted file mode 100644 index ff8ae2d19bd56..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/fetch_soon.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { estypes } from '@elastic/elasticsearch'; -import { UI_SETTINGS } from '../../../constants'; -import { FetchHandlers, SearchRequest } from '../fetch'; -import { ISearchOptions } from '../../index'; -import { callClient } from './call_client'; - -/** - * This function introduces a slight delay in the request process to allow multiple requests to queue - * up (e.g. when a dashboard is loading). - */ -export async function fetchSoon( - request: SearchRequest, - options: ISearchOptions, - fetchHandlers: FetchHandlers -) { - const msToDelay = fetchHandlers.getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES) ? 50 : 0; - return delayedFetch(request, options, fetchHandlers, msToDelay); -} - -/** - * Delays executing a function for a given amount of time, and returns a promise that resolves - * with the result. - * @param fn The function to invoke - * @param ms The number of milliseconds to wait - * @return Promise A promise that resolves with the result of executing the function - */ -function delay(fn: (...args: any) => T, ms: number): Promise { - return new Promise((resolve) => { - setTimeout(() => resolve(fn()), ms); - }); -} - -// The current batch/queue of requests to fetch -let requestsToFetch: SearchRequest[] = []; -let requestOptions: ISearchOptions[] = []; - -// The in-progress fetch (if there is one) -let fetchInProgress: any = null; - -/** - * Delay fetching for a given amount of time, while batching up the requests to be fetched. - * Returns a promise that resolves with the response for the given request. - * @param request The request to fetch - * @param ms The number of milliseconds to wait (and batch requests) - * @return Promise The response for the given request - */ -async function delayedFetch( - request: SearchRequest, - options: ISearchOptions, - fetchHandlers: FetchHandlers, - ms: number -): Promise> { - if (ms === 0) { - return callClient([request], [options], fetchHandlers)[0] as Promise< - estypes.SearchResponse - >; - } - - const i = requestsToFetch.length; - requestsToFetch = [...requestsToFetch, request]; - requestOptions = [...requestOptions, options]; - - // Note: the typescript here only worked because `SearchResponse` was `any` - // Since this code is legacy, I'm leaving the any here. - const responses: any[] = await (fetchInProgress = - fetchInProgress || - delay(() => { - const response = callClient(requestsToFetch, requestOptions, fetchHandlers); - requestsToFetch = []; - requestOptions = []; - fetchInProgress = null; - return response; - }, ms)); - return responses[i]; -} diff --git a/src/plugins/data/common/search/search_source/legacy/index.ts b/src/plugins/data/common/search/search_source/legacy/index.ts index 2c90dc6795423..12594660136d8 100644 --- a/src/plugins/data/common/search/search_source/legacy/index.ts +++ b/src/plugins/data/common/search/search_source/legacy/index.ts @@ -6,5 +6,4 @@ * Side Public License, v 1. */ -export { fetchSoon } from './fetch_soon'; export * from './types'; diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 7f8a4fceff05d..333d076747b56 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -8,18 +8,12 @@ import { BehaviorSubject, of } from 'rxjs'; import { IndexPattern } from '../../index_patterns'; -import { GetConfigFn } from '../../types'; -import { fetchSoon } from './legacy'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; import { AggConfigs, AggTypesRegistryStart } from '../../'; import { mockAggTypesRegistry } from '../aggs/test_helpers'; import { RequestResponder } from 'src/plugins/inspector/common'; import { switchMap } from 'rxjs/operators'; -jest.mock('./legacy', () => ({ - fetchSoon: jest.fn().mockResolvedValue({}), -})); - const getComputedFields = () => ({ storedFields: [], scriptFields: {}, @@ -863,23 +857,6 @@ describe('SearchSource', () => { }); describe('fetch$', () => { - describe('#legacy fetch()', () => { - beforeEach(() => { - searchSourceDependencies = { - ...searchSourceDependencies, - getConfig: jest.fn(() => { - return true; // batchSearches = true - }) as GetConfigFn, - }; - }); - - test('should call msearch', async () => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - await searchSource.fetch$(options).toPromise(); - expect(fetchSoon).toBeCalledTimes(1); - }); - }); describe('responses', () => { test('should return partial results', async () => { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index d7f7973ae936c..f7eda781351a8 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -96,7 +96,6 @@ import { IKibanaSearchResponse, } from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; -import { fetchSoon } from './legacy'; import { extractReferences } from './extract_references'; /** @internal */ @@ -274,8 +273,6 @@ export class SearchSource { * @param options */ fetch$(options: ISearchOptions = {}) { - const { getConfig } = this.dependencies; - const s$ = defer(() => this.requestIsStarting(options)).pipe( switchMap(() => { const searchRequest = this.flatten(); @@ -284,9 +281,7 @@ export class SearchSource { options.indexPattern = searchRequest.index; } - return getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES) - ? from(this.legacyFetch(searchRequest, options)) - : this.fetchSearch$(searchRequest, options); + return this.fetchSearch$(searchRequest, options); }), tap((response) => { // TODO: Remove casting when https://github.com/elastic/elasticsearch-js/issues/1287 is resolved @@ -480,27 +475,6 @@ export class SearchSource { ); } - /** - * Run a search using the search service - * @return {Promise>} - */ - private async legacyFetch(searchRequest: SearchRequest, options: ISearchOptions) { - const { getConfig, legacy, onResponse } = this.dependencies; - - return await fetchSoon( - searchRequest, - { - ...(this.searchStrategyId && { searchStrategyId: this.searchStrategyId }), - ...options, - }, - { - getConfig, - onResponse, - legacy, - } - ); - } - /** * Called by requests of this search source when they are started * @param options diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 83a44b6f68af6..aba7c8a7dd85e 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -53,6 +53,7 @@ import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; import { DataPublicPluginStart, DataStartDependencies } from '../types'; import { NowProviderInternalContract } from '../now_provider'; import { getKibanaContext } from './expressions/kibana_context'; +import { ES_SEARCH_STRATEGY, UI_SETTINGS } from '..'; /** @internal */ export interface SearchServiceSetupDependencies { @@ -157,10 +158,16 @@ export class SearchService implements Plugin { } public start( - { application, http, notifications, uiSettings }: CoreStart, + { http, uiSettings }: CoreStart, { fieldFormats, indexPatterns }: SearchServiceStartDependencies ): ISearchStart { - const search = ((request, options) => { + const search = ((request, options = {}) => { + // Use the sync search strategy if legacy search is enabled. + // This still uses bfetch for batching. + const useSyncSearch = uiSettings.get(UI_SETTINGS.COURIER_BATCH_SEARCHES); + if (!options?.strategy && useSyncSearch) { + options.strategy = ES_SEARCH_STRATEGY; + } return this.searchInterceptor.search(request, options); }) as ISearchGeneric; From 504d1da8ac75ea6aa66a14bf72d3adef2df41adf Mon Sep 17 00:00:00 2001 From: Liza K Date: Tue, 13 Apr 2021 13:04:30 +0300 Subject: [PATCH 13/36] ts --- .../data/common/search/search_source/search_source.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 333d076747b56..498400c03cbb2 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -857,7 +857,6 @@ describe('SearchSource', () => { }); describe('fetch$', () => { - describe('responses', () => { test('should return partial results', async () => { searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); From ce1a83a3c6de77e7d18cd6e2b5407edc8acc8177 Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 21 Apr 2021 11:27:07 +0300 Subject: [PATCH 14/36] override to sync search in search source --- .../search_source/search_source.test.ts | 32 ++++++++++++++++++- .../search/search_source/search_source.ts | 11 ++++++- .../data/public/search/search_service.ts | 7 ---- src/plugins/data/server/ui_settings.ts | 4 +-- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 375bab0265e10..a0d5c0d6bfd5a 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -8,8 +8,9 @@ import { BehaviorSubject, of } from 'rxjs'; import { IndexPattern } from '../../index_patterns'; +import { GetConfigFn } from '../../types'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; -import { AggConfigs, AggTypesRegistryStart } from '../../'; +import { AggConfigs, AggTypesRegistryStart, ES_SEARCH_STRATEGY } from '../../'; import { mockAggTypesRegistry } from '../aggs/test_helpers'; import { RequestResponder } from 'src/plugins/inspector/common'; import { switchMap } from 'rxjs/operators'; @@ -863,6 +864,35 @@ describe('SearchSource', () => { }); describe('fetch$', () => { + describe('#legacy COURIER_BATCH_SEARCHES', () => { + beforeEach(() => { + searchSourceDependencies = { + ...searchSourceDependencies, + getConfig: jest.fn(() => { + return true; // batchSearches = true + }) as GetConfigFn, + }; + }); + + test('should override to use sync search if not set', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + await searchSource.fetch$(options).toPromise(); + + const [_, callOptions] = mockSearchMethod.mock.calls[0]; + expect(callOptions.strategy).toBe(ES_SEARCH_STRATEGY); + }); + + test('should not override strategy if set ', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = { strategy: 'banana' }; + await searchSource.fetch$(options).toPromise(); + + const [_, callOptions] = mockSearchMethod.mock.calls[0]; + expect(callOptions.strategy).toBe('banana'); + }); + }); + describe('responses', () => { test('should return partial results', async () => { searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 02d948dbacb93..585126e1184d2 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -75,7 +75,7 @@ import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; -import { AggConfigs, ISearchGeneric, ISearchOptions } from '../..'; +import { AggConfigs, ES_SEARCH_STRATEGY, ISearchGeneric, ISearchOptions } from '../..'; import type { ISearchSource, SearchFieldValue, @@ -272,6 +272,15 @@ export class SearchSource { * @param options */ fetch$(options: ISearchOptions = {}) { + const { getConfig } = this.dependencies; + const syncSearchByDefault = getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES); + + // Use the sync search strategy if legacy search is enabled. + // This still uses bfetch for batching. + if (!options?.strategy && syncSearchByDefault) { + options.strategy = ES_SEARCH_STRATEGY; + } + const s$ = defer(() => this.requestIsStarting(options)).pipe( switchMap(() => { const searchRequest = this.flatten(); diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index aba7c8a7dd85e..98e87ef0195cd 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -53,7 +53,6 @@ import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; import { DataPublicPluginStart, DataStartDependencies } from '../types'; import { NowProviderInternalContract } from '../now_provider'; import { getKibanaContext } from './expressions/kibana_context'; -import { ES_SEARCH_STRATEGY, UI_SETTINGS } from '..'; /** @internal */ export interface SearchServiceSetupDependencies { @@ -162,12 +161,6 @@ export class SearchService implements Plugin { { fieldFormats, indexPatterns }: SearchServiceStartDependencies ): ISearchStart { const search = ((request, options = {}) => { - // Use the sync search strategy if legacy search is enabled. - // This still uses bfetch for batching. - const useSyncSearch = uiSettings.get(UI_SETTINGS.COURIER_BATCH_SEARCHES); - if (!options?.strategy && useSyncSearch) { - options.strategy = ES_SEARCH_STRATEGY; - } return this.searchInterceptor.search(request, options); }) as ISearchGeneric; diff --git a/src/plugins/data/server/ui_settings.ts b/src/plugins/data/server/ui_settings.ts index 971ae3bb7507b..78d7c15cac5d6 100644 --- a/src/plugins/data/server/ui_settings.ts +++ b/src/plugins/data/server/ui_settings.ts @@ -276,12 +276,12 @@ export function getUiSettings(): Record> { }, [UI_SETTINGS.COURIER_BATCH_SEARCHES]: { name: i18n.translate('data.advancedSettings.courier.batchSearchesTitle', { - defaultMessage: 'Use legacy search', + defaultMessage: 'Use sync search', }), value: false, type: 'boolean', description: i18n.translate('data.advancedSettings.courier.batchSearchesText', { - defaultMessage: `Kibana uses a new search and batching infrastructure. + defaultMessage: `Kibana uses a new asynchronous search and infrastructure. Enable this option if you prefer to fallback to the legacy synchronous behavior`, }), deprecation: { From c7a78277098304b4e605d8bb5d3d0aa3bff824ec Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 21 Apr 2021 11:51:57 +0300 Subject: [PATCH 15/36] delete more legacy code --- .../search/search_source/fetch/types.ts | 6 -- .../legacy/default_search_strategy.test.ts | 61 ------------------ .../legacy/default_search_strategy.ts | 63 ------------------- .../search/search_source/legacy/types.ts | 26 -------- .../search_source/search_source.test.ts | 4 +- src/plugins/data/public/public.api.md | 3 +- .../data/public/search/search_service.ts | 5 -- .../data/server/search/search_service.ts | 12 +--- src/plugins/data/server/server.api.md | 2 - 9 files changed, 5 insertions(+), 177 deletions(-) delete mode 100644 src/plugins/data/common/search/search_source/legacy/default_search_strategy.test.ts delete mode 100644 src/plugins/data/common/search/search_source/legacy/default_search_strategy.ts diff --git a/src/plugins/data/common/search/search_source/fetch/types.ts b/src/plugins/data/common/search/search_source/fetch/types.ts index 8e8a9f1025b80..79aa45163b913 100644 --- a/src/plugins/data/common/search/search_source/fetch/types.ts +++ b/src/plugins/data/common/search/search_source/fetch/types.ts @@ -7,7 +7,6 @@ */ import type { estypes } from '@elastic/elasticsearch'; -import { LegacyFetchHandlers } from '../legacy/types'; import { GetConfigFn } from '../../../types'; /** @@ -29,11 +28,6 @@ export interface FetchHandlers { request: SearchRequest, response: estypes.SearchResponse ) => estypes.SearchResponse; - /** - * These handlers are only used by the legacy defaultSearchStrategy and can be removed - * once that strategy has been deprecated. - */ - legacy: LegacyFetchHandlers; } export interface SearchError { diff --git a/src/plugins/data/common/search/search_source/legacy/default_search_strategy.test.ts b/src/plugins/data/common/search/search_source/legacy/default_search_strategy.test.ts deleted file mode 100644 index 9b03ebdbd116f..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/default_search_strategy.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { MockedKeys } from '@kbn/utility-types/jest'; -import { defaultSearchStrategy } from './default_search_strategy'; -import { LegacyFetchHandlers, SearchStrategySearchParams } from './types'; -import { BehaviorSubject } from 'rxjs'; - -const { search } = defaultSearchStrategy; - -describe('defaultSearchStrategy', () => { - describe('search', () => { - let searchArgs: MockedKeys; - - beforeEach(() => { - searchArgs = { - searchRequests: [ - { - index: { title: 'foo' }, - body: {}, - }, - ], - getConfig: jest.fn(), - onResponse: (req, res) => res, - legacy: { - callMsearch: jest.fn().mockResolvedValue(undefined), - loadingCount$: new BehaviorSubject(0) as any, - } as jest.Mocked, - }; - }); - - test('calls callMsearch with the correct arguments', async () => { - await search({ ...searchArgs }); - expect(searchArgs.legacy.callMsearch.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "body": Object { - "searches": Array [ - Object { - "body": Object {}, - "header": Object { - "index": "foo", - "preference": undefined, - }, - }, - ], - }, - "signal": AbortSignal {}, - }, - ], - ] - `); - }); - }); -}); diff --git a/src/plugins/data/common/search/search_source/legacy/default_search_strategy.ts b/src/plugins/data/common/search/search_source/legacy/default_search_strategy.ts deleted file mode 100644 index 16e109d65a5be..0000000000000 --- a/src/plugins/data/common/search/search_source/legacy/default_search_strategy.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { getPreference } from '../fetch'; -import { SearchStrategyProvider, SearchStrategySearchParams } from './types'; - -// @deprecated -export const defaultSearchStrategy: SearchStrategyProvider = { - id: 'default', - - search: (params) => { - return msearch(params); - }, -}; - -function msearch({ searchRequests, getConfig, legacy }: SearchStrategySearchParams) { - const { callMsearch, loadingCount$ } = legacy; - - const requests = searchRequests.map(({ index, body }) => { - return { - header: { - index: index.title || index, - preference: getPreference(getConfig), - }, - body, - }; - }); - - const abortController = new AbortController(); - let resolved = false; - - // Start LoadingIndicator - loadingCount$.next(loadingCount$.getValue() + 1); - - const cleanup = () => { - if (!resolved) { - resolved = true; - // Decrement loading counter & cleanup BehaviorSubject - loadingCount$.next(loadingCount$.getValue() - 1); - loadingCount$.complete(); - } - }; - - const searching = callMsearch({ - body: { searches: requests }, - signal: abortController.signal, - }) - .then((res: any) => res?.body?.responses) - .finally(() => cleanup()); - - return { - abort: () => { - abortController.abort(); - cleanup(); - }, - searching, - }; -} diff --git a/src/plugins/data/common/search/search_source/legacy/types.ts b/src/plugins/data/common/search/search_source/legacy/types.ts index a4328528fd662..6778be77c21c5 100644 --- a/src/plugins/data/common/search/search_source/legacy/types.ts +++ b/src/plugins/data/common/search/search_source/legacy/types.ts @@ -6,9 +6,7 @@ * Side Public License, v 1. */ -import { BehaviorSubject } from 'rxjs'; import type { estypes, ApiResponse } from '@elastic/elasticsearch'; -import { FetchHandlers, SearchRequest } from '../fetch'; interface MsearchHeaders { index: string; @@ -29,27 +27,3 @@ export interface MsearchRequestBody { export interface MsearchResponse { body: ApiResponse<{ responses: Array> }>; } - -// @internal -export interface LegacyFetchHandlers { - callMsearch: (params: { - body: MsearchRequestBody; - signal: AbortSignal; - }) => Promise; - loadingCount$: BehaviorSubject; -} - -export interface SearchStrategySearchParams extends FetchHandlers { - searchRequests: SearchRequest[]; -} - -// @deprecated -export interface SearchStrategyProvider { - id: string; - search: (params: SearchStrategySearchParams) => SearchStrategyResponse; -} - -export interface SearchStrategyResponse { - searching: Promise>>; - abort: () => void; -} diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index a0d5c0d6bfd5a..9fa6bcf9d8e33 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -879,7 +879,7 @@ describe('SearchSource', () => { const options = {}; await searchSource.fetch$(options).toPromise(); - const [_, callOptions] = mockSearchMethod.mock.calls[0]; + const [, callOptions] = mockSearchMethod.mock.calls[0]; expect(callOptions.strategy).toBe(ES_SEARCH_STRATEGY); }); @@ -888,7 +888,7 @@ describe('SearchSource', () => { const options = { strategy: 'banana' }; await searchSource.fetch$(options).toPromise(); - const [_, callOptions] = mockSearchMethod.mock.calls[0]; + const [, callOptions] = mockSearchMethod.mock.calls[0]; expect(callOptions.strategy).toBe('banana'); }); }); diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index dc138b7347d04..d34f159d7ec65 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -7,8 +7,7 @@ import { $Values } from '@kbn/utility-types'; import { Action } from 'history'; import { Adapters as Adapters_2 } from 'src/plugins/inspector/common'; -import { ApiResponse } from '@elastic/elasticsearch'; -import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch/lib/Transport'; +import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; import { ApplicationStart } from 'kibana/public'; import { Assign } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 98e87ef0195cd..ec7a486445b71 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -35,7 +35,6 @@ import { phraseFilterFunction, esRawResponse, } from '../../common/search'; -import { getCallMsearch } from './legacy'; import { AggsService, AggsStartDependencies } from './aggs'; import { IndexPatternsContract } from '../index_patterns/index_patterns'; import { ISearchInterceptor, SearchInterceptor } from './search_interceptor'; @@ -171,10 +170,6 @@ export class SearchService implements Plugin { getConfig: uiSettings.get.bind(uiSettings), search, onResponse: handleResponse, - legacy: { - callMsearch: getCallMsearch({ http }), - loadingCount$, - }, }; return { diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index e53244fa7ff26..cc4d069333eab 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { BehaviorSubject, from, Observable, throwError } from 'rxjs'; +import { from, Observable, throwError } from 'rxjs'; import { pick } from 'lodash'; import moment from 'moment'; import { @@ -36,7 +36,7 @@ import { AggsService } from './aggs'; import { FieldFormatsStart } from '../field_formats'; import { IndexPatternsServiceStart } from '../index_patterns'; -import { getCallMsearch, registerMsearchRoute, registerSearchRoute } from './routes'; +import { registerMsearchRoute, registerSearchRoute } from './routes'; import { ES_SEARCH_STRATEGY, esSearchStrategyProvider } from './es_search'; import { DataPluginStart, DataPluginStartDependencies } from '../plugin'; import { UsageCollectionSetup } from '../../../usage_collection/server'; @@ -226,14 +226,6 @@ export class SearchService implements Plugin { getConfig: (key: string): T => uiSettingsCache[key], search: this.asScoped(request).search, onResponse: (req, res) => res, - legacy: { - callMsearch: getCallMsearch({ - esClient, - globalConfig$: this.initializerContext.config.legacy.globalConfig$, - uiSettings: uiSettingsClient, - }), - loadingCount$: new BehaviorSubject(0), - }, }; return this.searchSourceService.start(scopedIndexPatterns, searchSourceDependencies); diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 3316e8102e50a..54d9e64fddb7f 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -6,9 +6,7 @@ import { $Values } from '@kbn/utility-types'; import { Adapters } from 'src/plugins/inspector/common'; -import { ApiResponse } from '@elastic/elasticsearch'; import { Assign } from '@kbn/utility-types'; -import { BehaviorSubject } from 'rxjs'; import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/server'; From ed547f14c4f481c0d688b6651c92dabee45fc4b7 Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 21 Apr 2021 12:12:36 +0300 Subject: [PATCH 16/36] ts --- .../search/search_source/create_search_source.test.ts | 5 ----- .../data/common/search/search_source/legacy/types.ts | 6 ++++++ src/plugins/data/common/search/search_source/mocks.ts | 6 +----- .../data/common/search/search_source/search_source.test.ts | 6 +----- .../search/search_source/search_source_service.test.ts | 5 ----- src/plugins/data/public/search/legacy/call_msearch.ts | 4 ++-- 6 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/plugins/data/common/search/search_source/create_search_source.test.ts b/src/plugins/data/common/search/search_source/create_search_source.test.ts index df31719b0aec4..6a6ac1dfa93e7 100644 --- a/src/plugins/data/common/search/search_source/create_search_source.test.ts +++ b/src/plugins/data/common/search/search_source/create_search_source.test.ts @@ -11,7 +11,6 @@ import { SearchSourceDependencies } from './search_source'; import { IIndexPattern } from '../../index_patterns'; import { IndexPatternsContract } from '../../index_patterns/index_patterns'; import { Filter } from '../../es_query/filters'; -import { BehaviorSubject } from 'rxjs'; describe('createSearchSource', () => { const indexPatternMock: IIndexPattern = {} as IIndexPattern; @@ -24,10 +23,6 @@ describe('createSearchSource', () => { getConfig: jest.fn(), search: jest.fn(), onResponse: (req, res) => res, - legacy: { - callMsearch: jest.fn(), - loadingCount$: new BehaviorSubject(0), - }, }; indexPatternContractMock = ({ diff --git a/src/plugins/data/common/search/search_source/legacy/types.ts b/src/plugins/data/common/search/search_source/legacy/types.ts index 6778be77c21c5..0425c43e5d9d8 100644 --- a/src/plugins/data/common/search/search_source/legacy/types.ts +++ b/src/plugins/data/common/search/search_source/legacy/types.ts @@ -27,3 +27,9 @@ export interface MsearchRequestBody { export interface MsearchResponse { body: ApiResponse<{ responses: Array> }>; } + +// @internal +export type CallMsearchFn = (params: { + body: MsearchRequestBody; + signal: AbortSignal; +}) => Promise; diff --git a/src/plugins/data/common/search/search_source/mocks.ts b/src/plugins/data/common/search/search_source/mocks.ts index ade22c20596d9..64ed82f36e81b 100644 --- a/src/plugins/data/common/search/search_source/mocks.ts +++ b/src/plugins/data/common/search/search_source/mocks.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { BehaviorSubject, of } from 'rxjs'; +import { of } from 'rxjs'; import type { MockedKeys } from '@kbn/utility-types/jest'; import { uiSettingsServiceMock } from '../../../../../core/public/mocks'; @@ -47,8 +47,4 @@ export const createSearchSourceMock = (fields?: SearchSourceFields) => getConfig: uiSettingsServiceMock.createStartContract().get, search: jest.fn(), onResponse: jest.fn().mockImplementation((req, res) => res), - legacy: { - callMsearch: jest.fn(), - loadingCount$: new BehaviorSubject(0), - }, }); diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 9fa6bcf9d8e33..68e386acfd48c 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { BehaviorSubject, of } from 'rxjs'; +import { of } from 'rxjs'; import { IndexPattern } from '../../index_patterns'; import { GetConfigFn } from '../../types'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; @@ -84,10 +84,6 @@ describe('SearchSource', () => { getConfig: getConfigMock, search: mockSearchMethod, onResponse: (req, res) => res, - legacy: { - callMsearch: jest.fn(), - loadingCount$: new BehaviorSubject(0), - }, }; searchSource = new SearchSource({}, searchSourceDependencies); diff --git a/src/plugins/data/common/search/search_source/search_source_service.test.ts b/src/plugins/data/common/search/search_source/search_source_service.test.ts index 9e36d3c6002da..23bb809092bde 100644 --- a/src/plugins/data/common/search/search_source/search_source_service.test.ts +++ b/src/plugins/data/common/search/search_source/search_source_service.test.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { BehaviorSubject } from 'rxjs'; import { IndexPatternsContract } from '../../index_patterns/index_patterns'; import { SearchSourceService, SearchSourceDependencies } from './'; @@ -19,10 +18,6 @@ describe('SearchSource service', () => { getConfig: jest.fn(), search: jest.fn(), onResponse: jest.fn(), - legacy: { - callMsearch: jest.fn(), - loadingCount$: new BehaviorSubject(0), - }, }; }); diff --git a/src/plugins/data/public/search/legacy/call_msearch.ts b/src/plugins/data/public/search/legacy/call_msearch.ts index f20ae322fee57..ed6da13134d40 100644 --- a/src/plugins/data/public/search/legacy/call_msearch.ts +++ b/src/plugins/data/public/search/legacy/call_msearch.ts @@ -7,7 +7,7 @@ */ import { HttpStart } from 'src/core/public'; -import { LegacyFetchHandlers } from '../../../common/search/search_source'; +import { CallMsearchFn } from '../../../common/search/search_source'; /** * Wrapper for calling the internal msearch endpoint from the client. @@ -16,7 +16,7 @@ import { LegacyFetchHandlers } from '../../../common/search/search_source'; * * @internal */ -export function getCallMsearch({ http }: { http: HttpStart }): LegacyFetchHandlers['callMsearch'] { +export function getCallMsearch({ http }: { http: HttpStart }): CallMsearchFn { return async ({ body, signal }) => { return http.post('/internal/_msearch', { body: JSON.stringify(body), From c10110fe7ebb1729a1cc517afde5606104ca46fe Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 21 Apr 2021 12:21:33 +0300 Subject: [PATCH 17/36] delete moarrrr --- .../search/search_source/legacy/types.ts | 6 --- .../public/search/legacy/call_msearch.test.ts | 43 ------------------- .../data/public/search/legacy/call_msearch.ts | 26 ----------- .../data/public/search/legacy/index.ts | 9 ---- 4 files changed, 84 deletions(-) delete mode 100644 src/plugins/data/public/search/legacy/call_msearch.test.ts delete mode 100644 src/plugins/data/public/search/legacy/call_msearch.ts delete mode 100644 src/plugins/data/public/search/legacy/index.ts diff --git a/src/plugins/data/common/search/search_source/legacy/types.ts b/src/plugins/data/common/search/search_source/legacy/types.ts index 0425c43e5d9d8..6778be77c21c5 100644 --- a/src/plugins/data/common/search/search_source/legacy/types.ts +++ b/src/plugins/data/common/search/search_source/legacy/types.ts @@ -27,9 +27,3 @@ export interface MsearchRequestBody { export interface MsearchResponse { body: ApiResponse<{ responses: Array> }>; } - -// @internal -export type CallMsearchFn = (params: { - body: MsearchRequestBody; - signal: AbortSignal; -}) => Promise; diff --git a/src/plugins/data/public/search/legacy/call_msearch.test.ts b/src/plugins/data/public/search/legacy/call_msearch.test.ts deleted file mode 100644 index 0627a09e12e67..0000000000000 --- a/src/plugins/data/public/search/legacy/call_msearch.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { HttpStart } from 'src/core/public'; -import { coreMock } from '../../../../../core/public/mocks'; -import { getCallMsearch } from './call_msearch'; - -describe('callMsearch', () => { - const msearchMock = jest.fn().mockResolvedValue({ body: { responses: [] } }); - let http: jest.Mocked; - - beforeEach(() => { - msearchMock.mockClear(); - http = coreMock.createStart().http; - http.post.mockResolvedValue(msearchMock); - }); - - test('calls http.post with the correct arguments', async () => { - const searches = [{ header: { index: 'foo' }, body: {} }]; - const callMsearch = getCallMsearch({ http }); - await callMsearch({ - body: { searches }, - signal: new AbortController().signal, - }); - - expect(http.post.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - "/internal/_msearch", - Object { - "body": "{\\"searches\\":[{\\"header\\":{\\"index\\":\\"foo\\"},\\"body\\":{}}]}", - "signal": AbortSignal {}, - }, - ], - ] - `); - }); -}); diff --git a/src/plugins/data/public/search/legacy/call_msearch.ts b/src/plugins/data/public/search/legacy/call_msearch.ts deleted file mode 100644 index ed6da13134d40..0000000000000 --- a/src/plugins/data/public/search/legacy/call_msearch.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { HttpStart } from 'src/core/public'; -import { CallMsearchFn } from '../../../common/search/search_source'; - -/** - * Wrapper for calling the internal msearch endpoint from the client. - * This is needed to abstract away differences in the http service - * between client & server. - * - * @internal - */ -export function getCallMsearch({ http }: { http: HttpStart }): CallMsearchFn { - return async ({ body, signal }) => { - return http.post('/internal/_msearch', { - body: JSON.stringify(body), - signal, - }); - }; -} diff --git a/src/plugins/data/public/search/legacy/index.ts b/src/plugins/data/public/search/legacy/index.ts deleted file mode 100644 index 52f576d1b2e34..0000000000000 --- a/src/plugins/data/public/search/legacy/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export * from './call_msearch'; From af896c7723b3926739636649de7133566fe1abe6 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 14:47:13 +0300 Subject: [PATCH 18/36] deflate bfetch chunks --- package.json | 1 + .../batching/create_streaming_batched_function.ts | 10 +++++++++- .../bfetch/server/streaming/create_ndjson_stream.ts | 10 +++++++++- yarn.lock | 5 +++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 992433e17e6c1..1106a1e05f4b6 100644 --- a/package.json +++ b/package.json @@ -298,6 +298,7 @@ "p-limit": "^3.0.1", "p-map": "^4.0.0", "p-retry": "^4.2.0", + "pako": "^2.0.3", "papaparse": "^5.2.0", "pdfmake": "^0.1.65", "pegjs": "0.10.0", diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 2d81331f10a88..7b332178c21b3 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { inflate } from 'pako'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, @@ -127,10 +128,17 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; + const getResponse = (response: string) => { + const { compressed, payload } = JSON.parse(response); + const inputBuf = Buffer.from(payload, 'base64'); + const inflatedRes = compressed ? inflate(inputBuf, { to: 'string' }) : payload; + return JSON.parse(inflatedRes) as BatchResponseItem; + }; + stream.pipe(split('\n')).subscribe({ next: (json: string) => { try { - const response = JSON.parse(json) as BatchResponseItem; + const response = getResponse(json); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts index a237e5597162e..944854e4532c3 100644 --- a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts @@ -9,8 +9,10 @@ import { Observable } from 'rxjs'; import { Logger } from 'src/core/server'; import { Stream, PassThrough } from 'stream'; +import { deflateSync } from 'zlib'; const delimiter = '\n'; +const ENCODE_THRESHOLD = 1000; export const createNDJSONStream = ( results: Observable, @@ -21,7 +23,13 @@ export const createNDJSONStream = ( results.subscribe({ next: (message: Response) => { try { - const line = JSON.stringify(message); + const strMessage = JSON.stringify(message); + const compressed = strMessage.length > ENCODE_THRESHOLD; + const payload = compressed ? deflateSync(strMessage).toString('base64') : strMessage; + const line = JSON.stringify({ + compressed, + payload, + }); stream.write(`${line}${delimiter}`); } catch (error) { logger.error('Could not serialize or stream a message.'); diff --git a/yarn.lock b/yarn.lock index f4d7684174967..2f2ebdc8a7a1c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20866,6 +20866,11 @@ pako@^1.0.5, pako@~1.0.2, pako@~1.0.5: resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.10.tgz#4328badb5086a426aa90f541977d4955da5c9732" integrity sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw== +pako@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.3.tgz#cdf475e31b678565251406de9e759196a0ea7a43" + integrity sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw== + papaparse@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.2.0.tgz#97976a1b135c46612773029153dc64995caa3b7b" From dcf5d2ecad699c858182fb04ccbd267a44d3f32d Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 17:03:21 +0300 Subject: [PATCH 19/36] update tests use only zlib --- package.json | 1 - src/plugins/bfetch/common/batch.ts | 5 + .../create_streaming_batched_function.test.ts | 150 ++++++++++++++---- .../create_streaming_batched_function.ts | 16 +- .../server/streaming/create_ndjson_stream.ts | 2 +- test/api_integration/apis/search/bsearch.ts | 20 ++- yarn.lock | 5 - 7 files changed, 154 insertions(+), 45 deletions(-) diff --git a/package.json b/package.json index 1106a1e05f4b6..992433e17e6c1 100644 --- a/package.json +++ b/package.json @@ -298,7 +298,6 @@ "p-limit": "^3.0.1", "p-map": "^4.0.0", "p-retry": "^4.2.0", - "pako": "^2.0.3", "papaparse": "^5.2.0", "pdfmake": "^0.1.65", "pegjs": "0.10.0", diff --git a/src/plugins/bfetch/common/batch.ts b/src/plugins/bfetch/common/batch.ts index a84d94b541ae5..59b012751c66d 100644 --- a/src/plugins/bfetch/common/batch.ts +++ b/src/plugins/bfetch/common/batch.ts @@ -19,3 +19,8 @@ export interface BatchResponseItem): Promise<'resolved' | 'rejected' | 'pending'> => Promise.race<'resolved' | 'rejected' | 'pending'>([ @@ -259,8 +260,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { foo: 'bar' }, + compressed: false, + payload: { + id: 1, + result: { foo: 'bar' }, + }, }) + '\n' ); @@ -270,8 +274,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 0, - result: { foo: 'bar 2' }, + compressed: false, + payload: { + id: 0, + result: { foo: 'bar 2' }, + }, }) + '\n' ); @@ -296,14 +303,64 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { foo: 'bar' }, + compressed: false, + payload: { + id: 1, + result: { foo: 'bar' }, + }, }) + '\n' ); stream.next( JSON.stringify({ - id: 2, - result: { foo: 'bar 2' }, + compressed: false, + payload: { + id: 2, + result: { foo: 'bar 2' }, + }, + }) + '\n' + ); + + expect(await isPending(promise1)).toBe(true); + expect(await isPending(promise2)).toBe(false); + expect(await isPending(promise3)).toBe(false); + expect(await promise2).toEqual({ foo: 'bar' }); + expect(await promise3).toEqual({ foo: 'bar 2' }); + }); + + test('handles compressed chunks', async () => { + const { fetchStreaming, stream } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + fetchStreaming, + maxItemAge: 5, + flushOnMaxItems: 3, + }); + + const promise1 = fn({ a: '1' }); + const promise2 = fn({ b: '2' }); + const promise3 = fn({ c: '3' }); + await new Promise((r) => setTimeout(r, 6)); + + const compress = (resp: object) => { + return deflateSync(JSON.stringify(resp)).toString('base64'); + }; + + stream.next( + JSON.stringify({ + compressed: true, + payload: compress({ + id: 1, + result: { foo: 'bar' }, + }), + }) + '\n' + ); + stream.next( + JSON.stringify({ + compressed: true, + payload: compress({ + id: 2, + result: { foo: 'bar 2' }, + }), }) + '\n' ); @@ -330,20 +387,29 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 0, - result: false, + compressed: false, + payload: { + id: 0, + result: false, + }, }) + '\n' ); stream.next( JSON.stringify({ - id: 1, - result: 0, + compressed: false, + payload: { + id: 1, + result: 0, + }, }) + '\n' ); stream.next( JSON.stringify({ - id: 2, - result: '', + compressed: false, + payload: { + id: 2, + result: '', + }, }) + '\n' ); @@ -371,8 +437,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 0, - error: { message: 'oops' }, + compressed: false, + payload: { + id: 0, + error: { message: 'oops' }, + }, }) + '\n' ); @@ -400,8 +469,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 2, - result: { b: '3' }, + compressed: false, + payload: { + id: 2, + result: { b: '3' }, + }, }) + '\n' ); @@ -409,8 +481,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - error: { b: '2' }, + compressed: false, + payload: { + id: 1, + error: { b: '2' }, + }, }) + '\n' ); @@ -418,8 +493,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 0, - result: { b: '1' }, + compressed: false, + payload: { + id: 0, + result: { b: '1' }, + }, }) + '\n' ); @@ -489,8 +567,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { b: '2' }, + compressed: false, + payload: { + id: 1, + result: { b: '2' }, + }, }) + '\n' ); @@ -548,8 +629,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { b: '1' }, + compressed: false, + payload: { + id: 1, + result: { b: '1' }, + }, }) + '\n' ); stream.complete(); @@ -617,8 +701,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { b: '1' }, + compressed: false, + payload: { + id: 1, + result: { b: '1' }, + }, }) + '\n' ); stream.error('oops'); @@ -653,8 +740,11 @@ describe('createStreamingBatchedFunction()', () => { stream.next( JSON.stringify({ - id: 1, - result: { b: '1' }, + compressed: false, + payload: { + id: 1, + result: { b: '1' }, + }, }) + '\n' ); diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 7b332178c21b3..e9826e9894eea 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { inflate } from 'pako'; +import { inflateSync } from 'zlib'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, @@ -14,6 +14,7 @@ import { createBatchedFunction, BatchResponseItem, ErrorLike, + BatchItemWrapper, } from '../../common'; import { fetchStreaming, split } from '../streaming'; import { normalizeError } from '../../common'; @@ -129,10 +130,15 @@ export const createStreamingBatchedFunction = ( }; const getResponse = (response: string) => { - const { compressed, payload } = JSON.parse(response); - const inputBuf = Buffer.from(payload, 'base64'); - const inflatedRes = compressed ? inflate(inputBuf, { to: 'string' }) : payload; - return JSON.parse(inflatedRes) as BatchResponseItem; + const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; + + try { + const inputBuf = Buffer.from(payload, 'base64'); + const inflatedRes = compressed ? inflateSync(inputBuf).toString() : payload; + return JSON.parse(inflatedRes) as BatchResponseItem; + } catch (e) { + return payload; + } }; stream.pipe(split('\n')).subscribe({ diff --git a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts index 944854e4532c3..29012ebf67d4e 100644 --- a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts @@ -12,7 +12,7 @@ import { Stream, PassThrough } from 'stream'; import { deflateSync } from 'zlib'; const delimiter = '\n'; -const ENCODE_THRESHOLD = 1000; +const ENCODE_THRESHOLD = 1400; export const createNDJSONStream = ( results: Observable, diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index d0322624778ae..adca338941036 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -8,15 +8,25 @@ import expect from '@kbn/expect'; import request from 'superagent'; +import { inflateSync } from 'zlib'; import { FtrProviderContext } from '../../ftr_provider_context'; import { painlessErrReq } from './painless_err_req'; import { verifyErrorResponse } from './verify_error'; +const inflate = (resp: any) => { + if (resp.compressed) { + const inputBuf = Buffer.from(resp.payload, 'base64'); + return JSON.parse(inflateSync(inputBuf).toString()); + } else { + return JSON.parse(resp.payload); + } +}; + function parseBfetchResponse(resp: request.Response): Array> { return resp.text .trim() .split('\n') - .map((item) => JSON.parse(item)); + .map((item) => inflate(JSON.parse(item))); } export default function ({ getService }: FtrProviderContext) { @@ -33,7 +43,9 @@ export default function ({ getService }: FtrProviderContext) { params: { body: { query: { - match_all: {}, + bool: { + must: [{ match: { name: 'John' } }], + }, }, }, }, @@ -42,7 +54,9 @@ export default function ({ getService }: FtrProviderContext) { ], }); - const jsonBody = JSON.parse(resp.text); + const response = JSON.parse(resp.text); + expect(response.compressed).to.be(false); + const jsonBody = inflate(response); expect(resp.status).to.be(200); expect(jsonBody.id).to.be(0); diff --git a/yarn.lock b/yarn.lock index 2f2ebdc8a7a1c..f4d7684174967 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20866,11 +20866,6 @@ pako@^1.0.5, pako@~1.0.2, pako@~1.0.5: resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.10.tgz#4328badb5086a426aa90f541977d4955da5c9732" integrity sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw== -pako@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.3.tgz#cdf475e31b678565251406de9e759196a0ea7a43" - integrity sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw== - papaparse@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.2.0.tgz#97976a1b135c46612773029153dc64995caa3b7b" From 2d6192f92e45fac727794fed89e1dc34cf4baec3 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 17:30:36 +0300 Subject: [PATCH 20/36] ts --- .../public/batching/create_streaming_batched_function.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index e9826e9894eea..ea02df91216f3 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -129,15 +129,15 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; - const getResponse = (response: string) => { + const getResponse = (response: string): BatchResponseItem => { const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; try { const inputBuf = Buffer.from(payload, 'base64'); const inflatedRes = compressed ? inflateSync(inputBuf).toString() : payload; - return JSON.parse(inflatedRes) as BatchResponseItem; + return JSON.parse(inflatedRes); } catch (e) { - return payload; + return JSON.parse(payload); } }; From 44416cb9041e98d5fc9bc12dffb498251dcbef5c Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 18:36:10 +0300 Subject: [PATCH 21/36] extract getInflatedResponse --- .../create_streaming_batched_function.ts | 18 +------- .../public/batching/get_inflated_response.ts | 24 +++++++++++ test/api_integration/apis/search/bsearch.ts | 41 +++++++++---------- 3 files changed, 45 insertions(+), 38 deletions(-) create mode 100644 src/plugins/bfetch/public/batching/get_inflated_response.ts diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index ea02df91216f3..212e9f4021d16 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,19 +6,17 @@ * Side Public License, v 1. */ -import { inflateSync } from 'zlib'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, TimedItemBufferParams, createBatchedFunction, - BatchResponseItem, ErrorLike, - BatchItemWrapper, } from '../../common'; import { fetchStreaming, split } from '../streaming'; import { normalizeError } from '../../common'; import { BatchedFunc, BatchItem } from './types'; +import { getInflatedResponse } from './get_inflated_response'; export interface BatchedFunctionProtocolError extends ErrorLike { code: string; @@ -129,22 +127,10 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; - const getResponse = (response: string): BatchResponseItem => { - const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; - - try { - const inputBuf = Buffer.from(payload, 'base64'); - const inflatedRes = compressed ? inflateSync(inputBuf).toString() : payload; - return JSON.parse(inflatedRes); - } catch (e) { - return JSON.parse(payload); - } - }; - stream.pipe(split('\n')).subscribe({ next: (json: string) => { try { - const response = getResponse(json); + const response = getInflatedResponse(json); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/public/batching/get_inflated_response.ts b/src/plugins/bfetch/public/batching/get_inflated_response.ts new file mode 100644 index 0000000000000..7f8ad205274ba --- /dev/null +++ b/src/plugins/bfetch/public/batching/get_inflated_response.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { inflateSync } from 'zlib'; +import { BatchResponseItem, ErrorLike, BatchItemWrapper } from '../../common'; + +export function getInflatedResponse( + response: string +): BatchResponseItem { + const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; + + try { + const inputBuf = Buffer.from(payload, 'base64'); + const inflatedRes = compressed ? inflateSync(inputBuf).toString() : payload; + return JSON.parse(inflatedRes); + } catch (e) { + return JSON.parse(payload); + } +} diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index adca338941036..18a2e1204435a 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -8,25 +8,18 @@ import expect from '@kbn/expect'; import request from 'superagent'; -import { inflateSync } from 'zlib'; +import { getInflatedResponse } from '../../../../src/plugins/bfetch/public/batching/get_inflated_response'; import { FtrProviderContext } from '../../ftr_provider_context'; import { painlessErrReq } from './painless_err_req'; import { verifyErrorResponse } from './verify_error'; -const inflate = (resp: any) => { - if (resp.compressed) { - const inputBuf = Buffer.from(resp.payload, 'base64'); - return JSON.parse(inflateSync(inputBuf).toString()); - } else { - return JSON.parse(resp.payload); - } -}; - -function parseBfetchResponse(resp: request.Response): Array> { +function parseBfetchResponse(resp: request.Response) { return resp.text .trim() .split('\n') - .map((item) => inflate(JSON.parse(item))); + .map((item) => { + return getInflatedResponse(item); + }); } export default function ({ getService }: FtrProviderContext) { @@ -41,28 +34,28 @@ export default function ({ getService }: FtrProviderContext) { { request: { params: { + index: '.kibana', body: { query: { - bool: { - must: [{ match: { name: 'John' } }], - }, + match_all: {}, }, }, }, }, + options: { + strategy: 'es', + }, }, ], }); - const response = JSON.parse(resp.text); - expect(response.compressed).to.be(false); - const jsonBody = inflate(response); + const jsonBody = parseBfetchResponse(resp); expect(resp.status).to.be(200); - expect(jsonBody.id).to.be(0); - expect(jsonBody.result.isPartial).to.be(false); - expect(jsonBody.result.isRunning).to.be(false); - expect(jsonBody.result).to.have.property('rawResponse'); + expect(jsonBody[0].id).to.be(0); + expect(jsonBody[0].result.isPartial).to.be(false); + expect(jsonBody[0].result.isRunning).to.be(false); + expect(jsonBody[0].result).to.have.property('rawResponse'); }); it('should return a batch of successful resposes', async () => { @@ -71,6 +64,7 @@ export default function ({ getService }: FtrProviderContext) { { request: { params: { + index: '.kibana', body: { query: { match_all: {}, @@ -82,6 +76,7 @@ export default function ({ getService }: FtrProviderContext) { { request: { params: { + index: '.kibana', body: { query: { match_all: {}, @@ -109,6 +104,7 @@ export default function ({ getService }: FtrProviderContext) { { request: { params: { + index: '.kibana', body: { query: { match_all: {}, @@ -135,6 +131,7 @@ export default function ({ getService }: FtrProviderContext) { batch: [ { request: { + index: '.kibana', indexType: 'baad', params: { body: { From 2ad860e7ed809fd3d69176eb7ea856b85875c6b8 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 19:07:17 +0300 Subject: [PATCH 22/36] tests --- .../create_streaming_batched_function.test.ts | 200 +++++++----------- .../public/batching/get_inflated_response.ts | 5 +- 2 files changed, 84 insertions(+), 121 deletions(-) diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index dcd6eb9e6f650..c18f4ab4bb0fb 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -12,6 +12,17 @@ import { AbortError, defer, of } from '../../../kibana_utils/public'; import { Subject } from 'rxjs'; import { deflateSync } from 'zlib'; +const formatResponse = (resp: any, compressed: boolean = false) => { + return ( + JSON.stringify({ + compressed, + payload: compressed + ? deflateSync(JSON.stringify(resp)).toString('base64') + : JSON.stringify(resp), + }) + '\n' + ); +}; + const getPromiseState = (promise: Promise): Promise<'resolved' | 'rejected' | 'pending'> => Promise.race<'resolved' | 'rejected' | 'pending'>([ new Promise((resolve) => @@ -259,13 +270,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise3)).toBe(true); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { foo: 'bar' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { foo: 'bar' }, + }) ); expect(await isPending(promise1)).toBe(true); @@ -273,13 +281,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise3)).toBe(true); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 0, - result: { foo: 'bar 2' }, - }, - }) + '\n' + formatResponse({ + id: 0, + result: { foo: 'bar 2' }, + }) ); expect(await isPending(promise1)).toBe(false); @@ -302,22 +307,16 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { foo: 'bar' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { foo: 'bar' }, + }) ); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 2, - result: { foo: 'bar 2' }, - }, - }) + '\n' + formatResponse({ + id: 2, + result: { foo: 'bar 2' }, + }) ); expect(await isPending(promise1)).toBe(true); @@ -341,27 +340,23 @@ describe('createStreamingBatchedFunction()', () => { const promise3 = fn({ c: '3' }); await new Promise((r) => setTimeout(r, 6)); - const compress = (resp: object) => { - return deflateSync(JSON.stringify(resp)).toString('base64'); - }; - stream.next( - JSON.stringify({ - compressed: true, - payload: compress({ + formatResponse( + { id: 1, result: { foo: 'bar' }, - }), - }) + '\n' + }, + true + ) ); stream.next( - JSON.stringify({ - compressed: true, - payload: compress({ + formatResponse( + { id: 2, result: { foo: 'bar 2' }, - }), - }) + '\n' + }, + true + ) ); expect(await isPending(promise1)).toBe(true); @@ -386,31 +381,22 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 0, - result: false, - }, - }) + '\n' + formatResponse({ + id: 0, + result: false, + }) ); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: 0, - }, - }) + '\n' + formatResponse({ + id: 1, + result: 0, + }) ); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 2, - result: '', - }, - }) + '\n' + formatResponse({ + id: 2, + result: '', + }) ); expect(await isPending(promise1)).toBe(false); @@ -436,13 +422,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise)).toBe(true); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 0, - error: { message: 'oops' }, - }, - }) + '\n' + formatResponse({ + id: 0, + error: { message: 'oops' }, + }) ); expect(await isPending(promise)).toBe(false); @@ -468,37 +451,28 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 2, - result: { b: '3' }, - }, - }) + '\n' + formatResponse({ + id: 2, + result: { b: '3' }, + }) ); await new Promise((r) => setTimeout(r, 1)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - error: { b: '2' }, - }, - }) + '\n' + formatResponse({ + id: 1, + error: { b: '2' }, + }) ); await new Promise((r) => setTimeout(r, 1)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 0, - result: { b: '1' }, - }, - }) + '\n' + formatResponse({ + id: 0, + result: { b: '1' }, + }) ); await new Promise((r) => setTimeout(r, 1)); @@ -566,13 +540,10 @@ describe('createStreamingBatchedFunction()', () => { expect(error).toBeInstanceOf(AbortError); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { b: '2' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { b: '2' }, + }) ); await new Promise((r) => setTimeout(r, 1)); @@ -628,13 +599,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { b: '1' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { b: '1' }, + }) ); stream.complete(); @@ -700,13 +668,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { b: '1' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { b: '1' }, + }) ); stream.error('oops'); @@ -739,13 +704,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - JSON.stringify({ - compressed: false, - payload: { - id: 1, - result: { b: '1' }, - }, - }) + '\n' + formatResponse({ + id: 1, + result: { b: '1' }, + }) ); stream.next('Not a JSON\n'); diff --git a/src/plugins/bfetch/public/batching/get_inflated_response.ts b/src/plugins/bfetch/public/batching/get_inflated_response.ts index 7f8ad205274ba..90689613c7315 100644 --- a/src/plugins/bfetch/public/batching/get_inflated_response.ts +++ b/src/plugins/bfetch/public/batching/get_inflated_response.ts @@ -15,8 +15,9 @@ export function getInflatedResponse( const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; try { - const inputBuf = Buffer.from(payload, 'base64'); - const inflatedRes = compressed ? inflateSync(inputBuf).toString() : payload; + const inflatedRes = compressed + ? inflateSync(Buffer.from(payload, 'base64')).toString() + : payload; return JSON.parse(inflatedRes); } catch (e) { return JSON.parse(payload); From 8397d3df2c706e6aeaddb2929861f48792a4e2b8 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 22 Apr 2021 22:40:48 +0300 Subject: [PATCH 23/36] Use fflate in attempt to reduce package size --- package.json | 1 + .../common/compress/deflate_response.ts | 22 +++++++++++++++++++ src/plugins/bfetch/common/compress/index.ts | 10 +++++++++ .../compress/inflate_response.ts} | 6 ++--- src/plugins/bfetch/common/index.ts | 1 + .../create_streaming_batched_function.test.ts | 11 ++-------- .../create_streaming_batched_function.ts | 6 ++--- .../server/streaming/create_ndjson_stream.ts | 11 ++-------- test/api_integration/apis/search/bsearch.ts | 4 ++-- yarn.lock | 5 +++++ 10 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 src/plugins/bfetch/common/compress/deflate_response.ts create mode 100644 src/plugins/bfetch/common/compress/index.ts rename src/plugins/bfetch/{public/batching/get_inflated_response.ts => common/compress/inflate_response.ts} (81%) diff --git a/package.json b/package.json index 992433e17e6c1..0c0af6e2f339e 100644 --- a/package.json +++ b/package.json @@ -215,6 +215,7 @@ "expiry-js": "0.1.7", "extract-zip": "^2.0.1", "fast-deep-equal": "^3.1.1", + "fflate": "^0.6.9", "file-saver": "^1.3.8", "file-type": "^10.9.0", "focus-trap-react": "^3.1.1", diff --git a/src/plugins/bfetch/common/compress/deflate_response.ts b/src/plugins/bfetch/common/compress/deflate_response.ts new file mode 100644 index 0000000000000..ab777c3f71319 --- /dev/null +++ b/src/plugins/bfetch/common/compress/deflate_response.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { zlibSync, strToU8 } from 'fflate'; + +const ENCODE_THRESHOLD = 1400; + +export function deflateResponse(resp: T, compressed?: boolean) { + const strMessage = JSON.stringify(resp); + compressed = compressed ?? strMessage.length > ENCODE_THRESHOLD; + return JSON.stringify({ + compressed, + payload: compressed + ? Buffer.from(zlibSync(strToU8(strMessage), {})).toString('base64') + : strMessage, + }); +} diff --git a/src/plugins/bfetch/common/compress/index.ts b/src/plugins/bfetch/common/compress/index.ts new file mode 100644 index 0000000000000..0325fed343aaa --- /dev/null +++ b/src/plugins/bfetch/common/compress/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { deflateResponse } from './deflate_response'; +export { inflateResponse } from './inflate_response'; diff --git a/src/plugins/bfetch/public/batching/get_inflated_response.ts b/src/plugins/bfetch/common/compress/inflate_response.ts similarity index 81% rename from src/plugins/bfetch/public/batching/get_inflated_response.ts rename to src/plugins/bfetch/common/compress/inflate_response.ts index 90689613c7315..839d89e7e0c5b 100644 --- a/src/plugins/bfetch/public/batching/get_inflated_response.ts +++ b/src/plugins/bfetch/common/compress/inflate_response.ts @@ -6,17 +6,17 @@ * Side Public License, v 1. */ -import { inflateSync } from 'zlib'; +import { unzlibSync, strFromU8 } from 'fflate'; import { BatchResponseItem, ErrorLike, BatchItemWrapper } from '../../common'; -export function getInflatedResponse( +export function inflateResponse( response: string ): BatchResponseItem { const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; try { const inflatedRes = compressed - ? inflateSync(Buffer.from(payload, 'base64')).toString() + ? strFromU8(unzlibSync(Buffer.from(payload, 'base64'))) : payload; return JSON.parse(inflatedRes); } catch (e) { diff --git a/src/plugins/bfetch/common/index.ts b/src/plugins/bfetch/common/index.ts index deca56989722a..e0a441fd68477 100644 --- a/src/plugins/bfetch/common/index.ts +++ b/src/plugins/bfetch/common/index.ts @@ -10,3 +10,4 @@ export * from './util'; export * from './streaming'; export * from './buffer'; export * from './batch'; +export * from './compress'; diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index c18f4ab4bb0fb..e2ef70abe8e76 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -10,17 +10,10 @@ import { createStreamingBatchedFunction } from './create_streaming_batched_funct import { fetchStreaming as fetchStreamingReal } from '../streaming/fetch_streaming'; import { AbortError, defer, of } from '../../../kibana_utils/public'; import { Subject } from 'rxjs'; -import { deflateSync } from 'zlib'; +import { deflateResponse } from '../../common'; const formatResponse = (resp: any, compressed: boolean = false) => { - return ( - JSON.stringify({ - compressed, - payload: compressed - ? deflateSync(JSON.stringify(resp)).toString('base64') - : JSON.stringify(resp), - }) + '\n' - ); + return deflateResponse(resp, compressed) + '\n'; }; const getPromiseState = (promise: Promise): Promise<'resolved' | 'rejected' | 'pending'> => diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 212e9f4021d16..e78be3a109329 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -12,11 +12,11 @@ import { TimedItemBufferParams, createBatchedFunction, ErrorLike, + normalizeError, + inflateResponse, } from '../../common'; import { fetchStreaming, split } from '../streaming'; -import { normalizeError } from '../../common'; import { BatchedFunc, BatchItem } from './types'; -import { getInflatedResponse } from './get_inflated_response'; export interface BatchedFunctionProtocolError extends ErrorLike { code: string; @@ -130,7 +130,7 @@ export const createStreamingBatchedFunction = ( stream.pipe(split('\n')).subscribe({ next: (json: string) => { try { - const response = getInflatedResponse(json); + const response = inflateResponse(json); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts index 29012ebf67d4e..fa1f7e0763270 100644 --- a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts @@ -9,10 +9,9 @@ import { Observable } from 'rxjs'; import { Logger } from 'src/core/server'; import { Stream, PassThrough } from 'stream'; -import { deflateSync } from 'zlib'; +import { deflateResponse } from '../../common'; const delimiter = '\n'; -const ENCODE_THRESHOLD = 1400; export const createNDJSONStream = ( results: Observable, @@ -23,13 +22,7 @@ export const createNDJSONStream = ( results.subscribe({ next: (message: Response) => { try { - const strMessage = JSON.stringify(message); - const compressed = strMessage.length > ENCODE_THRESHOLD; - const payload = compressed ? deflateSync(strMessage).toString('base64') : strMessage; - const line = JSON.stringify({ - compressed, - payload, - }); + const line = deflateResponse(message); stream.write(`${line}${delimiter}`); } catch (error) { logger.error('Could not serialize or stream a message.'); diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index 18a2e1204435a..fea942779d60d 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import request from 'superagent'; -import { getInflatedResponse } from '../../../../src/plugins/bfetch/public/batching/get_inflated_response'; +import { inflateResponse } from '../../../../src/plugins/bfetch/common'; import { FtrProviderContext } from '../../ftr_provider_context'; import { painlessErrReq } from './painless_err_req'; import { verifyErrorResponse } from './verify_error'; @@ -18,7 +18,7 @@ function parseBfetchResponse(resp: request.Response) { .trim() .split('\n') .map((item) => { - return getInflatedResponse(item); + return inflateResponse(item); }); } diff --git a/yarn.lock b/yarn.lock index f4d7684174967..7b9411871507e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13362,6 +13362,11 @@ fetch-mock@^7.3.9: path-to-regexp "^2.2.1" whatwg-url "^6.5.0" +fflate@^0.6.9: + version "0.6.9" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.6.9.tgz#fb369b30792a03ff7274e174f3b36e51292d3f99" + integrity sha512-hmAdxNHub7fw36hX7BHiuAO0uekp6ufY2sjxBXWxIf0sw5p7tnS9GVrdM4D12SDYQUHVpiC50fPBYPTjOzRU2Q== + figgy-pudding@^3.5.1: version "3.5.1" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790" From a88372bd732a79120b0ab3d9652aa5727bd83fb7 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 2 May 2021 13:49:30 +0300 Subject: [PATCH 24/36] use node streams, fflate and hex encoding. --- packages/kbn-ui-shared-deps/entry.js | 2 + packages/kbn-ui-shared-deps/index.js | 1 + .../common/compress/inflate_response.ts | 13 ++-- .../create_streaming_batched_function.ts | 4 +- src/plugins/bfetch/server/plugin.ts | 6 +- .../streaming/create_compressed_stream.ts | 63 +++++++++++++++++++ .../server/streaming/create_ndjson_stream.ts | 3 +- src/plugins/bfetch/server/streaming/index.ts | 1 + 8 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 src/plugins/bfetch/server/streaming/create_compressed_stream.ts diff --git a/packages/kbn-ui-shared-deps/entry.js b/packages/kbn-ui-shared-deps/entry.js index 4029ce28faf5b..d3755ed7c5f29 100644 --- a/packages/kbn-ui-shared-deps/entry.js +++ b/packages/kbn-ui-shared-deps/entry.js @@ -44,6 +44,8 @@ export const Theme = require('./theme.ts'); export const Lodash = require('lodash'); export const LodashFp = require('lodash/fp'); +export const Fflate = require('fflate/esm/browser'); + // runtime deps which don't need to be copied across all bundles export const TsLib = require('tslib'); export const KbnAnalytics = require('@kbn/analytics'); diff --git a/packages/kbn-ui-shared-deps/index.js b/packages/kbn-ui-shared-deps/index.js index 62ddb09d25add..877bf3df6c039 100644 --- a/packages/kbn-ui-shared-deps/index.js +++ b/packages/kbn-ui-shared-deps/index.js @@ -52,6 +52,7 @@ exports.externals = { '@elastic/eui/dist/eui_theme_dark.json': '__kbnSharedDeps__.Theme.euiDarkVars', lodash: '__kbnSharedDeps__.Lodash', 'lodash/fp': '__kbnSharedDeps__.LodashFp', + fflate: '__kbnSharedDeps__.Fflate', /** * runtime deps which don't need to be copied across all bundles diff --git a/src/plugins/bfetch/common/compress/inflate_response.ts b/src/plugins/bfetch/common/compress/inflate_response.ts index 839d89e7e0c5b..f9cd15547de9a 100644 --- a/src/plugins/bfetch/common/compress/inflate_response.ts +++ b/src/plugins/bfetch/common/compress/inflate_response.ts @@ -7,19 +7,18 @@ */ import { unzlibSync, strFromU8 } from 'fflate'; -import { BatchResponseItem, ErrorLike, BatchItemWrapper } from '../../common'; +import { BatchResponseItem, ErrorLike } from '../../common'; export function inflateResponse( response: string ): BatchResponseItem { - const { compressed, payload } = JSON.parse(response) as BatchItemWrapper; - try { - const inflatedRes = compressed - ? strFromU8(unzlibSync(Buffer.from(payload, 'base64'))) - : payload; + const buff = Buffer.from(response, 'hex'); + + const unzip = unzlibSync(buff); + const inflatedRes = strFromU8(unzip); return JSON.parse(inflatedRes); } catch (e) { - return JSON.parse(payload); + return JSON.parse(response); } } diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index e78be3a109329..9610ffadcc161 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -127,10 +127,10 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; - stream.pipe(split('\n')).subscribe({ + stream.pipe(split('\r\n')).subscribe({ next: (json: string) => { try { - const response = inflateResponse(json); + const response = inflateResponse(json.trim()); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index 18f0813260f03..82da07df16e3a 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -28,7 +28,7 @@ import { normalizeError, } from '../common'; import { StreamingRequestHandler } from './types'; -import { createNDJSONStream } from './streaming'; +import { createCompressedStream } from './streaming'; // eslint-disable-next-line export interface BfetchServerSetupDependencies {} @@ -148,7 +148,7 @@ export class BfetchServerPlugin const data = request.body; return response.ok({ headers: streamingHeaders, - body: createNDJSONStream(handlerInstance.getResponseStream(data), logger), + body: createCompressedStream(handlerInstance.getResponseStream(data), logger), }); } ); @@ -166,7 +166,7 @@ export class BfetchServerPlugin const response$ = await streamHandler(context, request); return response.ok({ headers: streamingHeaders, - body: createNDJSONStream(response$, logger), + body: createCompressedStream(response$, logger), }); }; diff --git a/src/plugins/bfetch/server/streaming/create_compressed_stream.ts b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts new file mode 100644 index 0000000000000..79fd1eeaeb73e --- /dev/null +++ b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Observable } from 'rxjs'; +import { catchError, concatMap, finalize } from 'rxjs/operators'; +import { Logger } from 'src/core/server'; +import { Stream, PassThrough } from 'stream'; +import { createDeflate, constants } from 'zlib'; + +const delimiter = '\n'; + +async function zipMessageToStream(output: PassThrough, message: string) { + return new Promise((resolve, reject) => { + const gz = createDeflate({ + flush: constants.Z_SYNC_FLUSH, + }); + gz.on('error', function (err) { + reject(err); + }); + gz.on('data', (data) => { + output.write(data.toString('hex')); + }); + // gz.pipe(output, { end: false }); + gz.end(Buffer.from(message)); + + Stream.finished(gz, {}, () => { + output.write(delimiter); + resolve(undefined); + }); + }); +} + +export const createCompressedStream = ( + results: Observable, + logger: Logger +): Stream => { + const output = new PassThrough(); + + const sub = results + .pipe( + concatMap((message: Response) => { + const strMessage = JSON.stringify(message); + return zipMessageToStream(output, strMessage); + }), + catchError((e) => { + logger.error('Could not serialize or stream a message.'); + logger.error(e); + throw e; + }), + finalize(() => { + output.end(); + sub.unsubscribe(); + }) + ) + .subscribe(); + + return output; +}; diff --git a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts index fa1f7e0763270..a237e5597162e 100644 --- a/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_ndjson_stream.ts @@ -9,7 +9,6 @@ import { Observable } from 'rxjs'; import { Logger } from 'src/core/server'; import { Stream, PassThrough } from 'stream'; -import { deflateResponse } from '../../common'; const delimiter = '\n'; @@ -22,7 +21,7 @@ export const createNDJSONStream = ( results.subscribe({ next: (message: Response) => { try { - const line = deflateResponse(message); + const line = JSON.stringify(message); stream.write(`${line}${delimiter}`); } catch (error) { logger.error('Could not serialize or stream a message.'); diff --git a/src/plugins/bfetch/server/streaming/index.ts b/src/plugins/bfetch/server/streaming/index.ts index 2c31cc329295d..18fdfb85547d1 100644 --- a/src/plugins/bfetch/server/streaming/index.ts +++ b/src/plugins/bfetch/server/streaming/index.ts @@ -7,3 +7,4 @@ */ export * from './create_ndjson_stream'; +export * from './create_compressed_stream'; From 83e6f6197f7f16e2a009cbc51e2b7daba660a18e Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 5 May 2021 14:54:06 +0300 Subject: [PATCH 25/36] DISABLE_SEARCH_COMPRESSION UI Settings Use base64 and async compression --- .../common/compress/inflate_response.ts | 2 +- src/plugins/bfetch/common/constants.ts | 9 ++++ src/plugins/bfetch/common/index.ts | 1 + .../create_streaming_batched_function.ts | 12 +++++- src/plugins/bfetch/public/plugin.ts | 5 ++- src/plugins/bfetch/server/plugin.ts | 41 ++++++++++++++++--- .../streaming/create_compressed_stream.ts | 27 +++++------- .../bfetch/server/streaming/create_stream.ts | 23 +++++++++++ src/plugins/bfetch/server/streaming/index.ts | 1 + src/plugins/bfetch/server/ui_settings.ts | 29 +++++++++++++ 10 files changed, 125 insertions(+), 25 deletions(-) create mode 100644 src/plugins/bfetch/common/constants.ts create mode 100644 src/plugins/bfetch/server/streaming/create_stream.ts create mode 100644 src/plugins/bfetch/server/ui_settings.ts diff --git a/src/plugins/bfetch/common/compress/inflate_response.ts b/src/plugins/bfetch/common/compress/inflate_response.ts index f9cd15547de9a..0590152316cef 100644 --- a/src/plugins/bfetch/common/compress/inflate_response.ts +++ b/src/plugins/bfetch/common/compress/inflate_response.ts @@ -13,7 +13,7 @@ export function inflateResponse( response: string ): BatchResponseItem { try { - const buff = Buffer.from(response, 'hex'); + const buff = Buffer.from(response, 'base64'); const unzip = unzlibSync(buff); const inflatedRes = strFromU8(unzip); diff --git a/src/plugins/bfetch/common/constants.ts b/src/plugins/bfetch/common/constants.ts new file mode 100644 index 0000000000000..711f08bb06f83 --- /dev/null +++ b/src/plugins/bfetch/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const DISABLE_SEARCH_COMPRESSION = 'bfetch:disableCompression'; diff --git a/src/plugins/bfetch/common/index.ts b/src/plugins/bfetch/common/index.ts index e0a441fd68477..cee47e14d06a6 100644 --- a/src/plugins/bfetch/common/index.ts +++ b/src/plugins/bfetch/common/index.ts @@ -11,3 +11,4 @@ export * from './streaming'; export * from './buffer'; export * from './batch'; export * from './compress'; +export * from './constants'; diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 9610ffadcc161..572841e4c24f7 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -47,6 +47,11 @@ export interface StreamingBatchedFunctionParams { * before sending the batch request. */ maxItemAge?: TimedItemBufferParams['maxItemAge']; + + /** + * Disabled zlib compression of response chunks. + */ + disabledCompression?: boolean; } /** @@ -64,6 +69,7 @@ export const createStreamingBatchedFunction = ( fetchStreaming: fetchStreamingInjected = fetchStreaming, flushOnMaxItems = 25, maxItemAge = 10, + disabledCompression = false, } = params; const [fn] = createBatchedFunction({ onCall: (payload: Payload, signal?: AbortSignal) => { @@ -127,10 +133,12 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; - stream.pipe(split('\r\n')).subscribe({ + stream.pipe(split('\n')).subscribe({ next: (json: string) => { try { - const response = inflateResponse(json.trim()); + const response = disabledCompression + ? JSON.parse(json) + : inflateResponse(json); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts index ed97d468eec0b..655527b309faf 100644 --- a/src/plugins/bfetch/public/plugin.ts +++ b/src/plugins/bfetch/public/plugin.ts @@ -8,7 +8,7 @@ import { CoreStart, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/public'; import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './streaming'; -import { removeLeadingSlash } from '../common'; +import { removeLeadingSlash, DISABLE_SEARCH_COMPRESSION } from '../common'; import { createStreamingBatchedFunction, StreamingBatchedFunctionParams, @@ -40,6 +40,7 @@ export class BfetchPublicPlugin BfetchPublicStartDependencies > { private contract!: BfetchPublicContract; + private disabledCompression!: boolean; constructor(private readonly initializerContext: PluginInitializerContext) {} @@ -59,6 +60,7 @@ export class BfetchPublicPlugin } public start(core: CoreStart, plugins: BfetchPublicStartDependencies): BfetchPublicStart { + this.disabledCompression = core.uiSettings.get(DISABLE_SEARCH_COMPRESSION, true); return this.contract; } @@ -84,5 +86,6 @@ export class BfetchPublicPlugin createStreamingBatchedFunction({ ...params, fetchStreaming: params.fetchStreaming || fetchStreaming, + disabledCompression: this.disabledCompression, }); } diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index 82da07df16e3a..ac2fd597bb216 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -16,6 +16,7 @@ import type { RouteMethod, RequestHandler, RequestHandlerContext, + StartServicesAccessor, } from 'src/core/server'; import { schema } from '@kbn/config-schema'; import { Subject } from 'rxjs'; @@ -28,7 +29,9 @@ import { normalizeError, } from '../common'; import { StreamingRequestHandler } from './types'; -import { createCompressedStream } from './streaming'; +import { createStream } from './streaming'; +import { getUiSettings } from './ui_settings'; +import { DISABLE_SEARCH_COMPRESSION } from '../common'; // eslint-disable-next-line export interface BfetchServerSetupDependencies {} @@ -112,9 +115,19 @@ export class BfetchServerPlugin public setup(core: CoreSetup, plugins: BfetchServerSetupDependencies): BfetchServerSetup { const logger = this.initializerContext.logger.get(); const router = core.http.createRouter(); - const addStreamingResponseRoute = this.addStreamingResponseRoute({ router, logger }); + + core.uiSettings.register(getUiSettings()); + + const addStreamingResponseRoute = this.addStreamingResponseRoute({ + getStartServices: core.getStartServices, + router, + logger, + }); const addBatchProcessingRoute = this.addBatchProcessingRoute(addStreamingResponseRoute); - const createStreamingRequestHandler = this.createStreamingRequestHandler({ logger }); + const createStreamingRequestHandler = this.createStreamingRequestHandler({ + getStartServices: core.getStartServices, + logger, + }); return { addBatchProcessingRoute, @@ -129,10 +142,23 @@ export class BfetchServerPlugin public stop() {} + private async getCompressionDisabled( + request: KibanaRequest, + getStartServices: StartServicesAccessor + ) { + const [core] = await getStartServices(); + const uiSettingsClient = core.uiSettings.asScopedToClient( + core.savedObjects.getScopedClient(request) + ); + return uiSettingsClient.get(DISABLE_SEARCH_COMPRESSION); + } + private addStreamingResponseRoute = ({ + getStartServices, router, logger, }: { + getStartServices: StartServicesAccessor; router: ReturnType; logger: Logger; }): BfetchServerSetup['addStreamingResponseRoute'] => (path, handler) => { @@ -144,11 +170,13 @@ export class BfetchServerPlugin }, }, async (context, request, response) => { + const compressionDisabled = await this.getCompressionDisabled(request, getStartServices); + const handlerInstance = handler(request); const data = request.body; return response.ok({ headers: streamingHeaders, - body: createCompressedStream(handlerInstance.getResponseStream(data), logger), + body: createStream(handlerInstance.getResponseStream(data), logger, compressionDisabled), }); } ); @@ -156,17 +184,20 @@ export class BfetchServerPlugin private createStreamingRequestHandler = ({ logger, + getStartServices, }: { logger: Logger; + getStartServices: StartServicesAccessor; }): BfetchServerSetup['createStreamingRequestHandler'] => (streamHandler) => async ( context, request, response ) => { const response$ = await streamHandler(context, request); + const compressionDisabled = await this.getCompressionDisabled(request, getStartServices); return response.ok({ headers: streamingHeaders, - body: createCompressedStream(response$, logger), + body: createStream(response$, logger, compressionDisabled), }); }; diff --git a/src/plugins/bfetch/server/streaming/create_compressed_stream.ts b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts index 79fd1eeaeb73e..582b430be1ba2 100644 --- a/src/plugins/bfetch/server/streaming/create_compressed_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts @@ -6,32 +6,27 @@ * Side Public License, v 1. */ +import { promisify } from 'util'; import { Observable } from 'rxjs'; import { catchError, concatMap, finalize } from 'rxjs/operators'; import { Logger } from 'src/core/server'; import { Stream, PassThrough } from 'stream'; -import { createDeflate, constants } from 'zlib'; +import { constants, deflate } from 'zlib'; const delimiter = '\n'; async function zipMessageToStream(output: PassThrough, message: string) { - return new Promise((resolve, reject) => { - const gz = createDeflate({ - flush: constants.Z_SYNC_FLUSH, - }); - gz.on('error', function (err) { - reject(err); - }); - gz.on('data', (data) => { - output.write(data.toString('hex')); - }); - // gz.pipe(output, { end: false }); - gz.end(Buffer.from(message)); - - Stream.finished(gz, {}, () => { + return new Promise(async (resolve, reject) => { + try { + const gzipped = await promisify(deflate)(message, { + flush: constants.Z_SYNC_FLUSH, + }); + output.write(gzipped.toString('base64')); output.write(delimiter); resolve(undefined); - }); + } catch (err) { + reject(err); + } }); } diff --git a/src/plugins/bfetch/server/streaming/create_stream.ts b/src/plugins/bfetch/server/streaming/create_stream.ts new file mode 100644 index 0000000000000..7d6981294341b --- /dev/null +++ b/src/plugins/bfetch/server/streaming/create_stream.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Logger } from 'kibana/server'; +import { Stream } from 'stream'; +import { Observable } from 'rxjs'; +import { createCompressedStream } from './create_compressed_stream'; +import { createNDJSONStream } from './create_ndjson_stream'; + +export function createStream( + response$: Observable, + logger: Logger, + compressionDisabled: boolean +): Stream { + return compressionDisabled + ? createNDJSONStream(response$, logger) + : createCompressedStream(response$, logger); +} diff --git a/src/plugins/bfetch/server/streaming/index.ts b/src/plugins/bfetch/server/streaming/index.ts index 18fdfb85547d1..dfd472b5034a1 100644 --- a/src/plugins/bfetch/server/streaming/index.ts +++ b/src/plugins/bfetch/server/streaming/index.ts @@ -8,3 +8,4 @@ export * from './create_ndjson_stream'; export * from './create_compressed_stream'; +export * from './create_stream'; diff --git a/src/plugins/bfetch/server/ui_settings.ts b/src/plugins/bfetch/server/ui_settings.ts new file mode 100644 index 0000000000000..2fb9268f368d8 --- /dev/null +++ b/src/plugins/bfetch/server/ui_settings.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { UiSettingsParams } from 'src/core/server'; +import { schema } from '@kbn/config-schema'; +import { DISABLE_SEARCH_COMPRESSION } from '../common'; + +export function getUiSettings(): Record> { + return { + [DISABLE_SEARCH_COMPRESSION]: { + name: i18n.translate('bfetch.disableSearchCompression', { + defaultMessage: 'Disable Search Compression', + }), + value: false, + description: i18n.translate('bfetch.disableSearchCompressionDesc', { + defaultMessage: + 'Disable search compression. This allows you debug individual search requests, but increases response size.', + }), + schema: schema.boolean(), + category: ['search'], + }, + }; +} From 23d8252eee870066e76aa88c926882b2bfc5a875 Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 5 May 2021 15:12:19 +0300 Subject: [PATCH 26/36] i18n --- .i18nrc.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.i18nrc.json b/.i18nrc.json index efbb5ecc0194e..26694461b3b22 100644 --- a/.i18nrc.json +++ b/.i18nrc.json @@ -4,6 +4,7 @@ "console": "src/plugins/console", "core": "src/core", "discover": "src/plugins/discover", + "bfetch": "src/plugins/bfetch", "dashboard": "src/plugins/dashboard", "data": "src/plugins/data", "embeddableApi": "src/plugins/embeddable", From ad6e056f2fc534a515948a93d1bfff011e29e26d Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 5 May 2021 17:25:15 +0300 Subject: [PATCH 27/36] Code review Use custom header for compression Promisify once --- .../create_streaming_batched_function.ts | 14 ++++++++++++-- src/plugins/bfetch/public/plugin.ts | 12 ++++++------ src/plugins/bfetch/server/plugin.ts | 16 ++++------------ .../server/streaming/create_compressed_stream.ts | 3 ++- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 572841e4c24f7..4d36be72ec145 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { StartServicesAccessor } from 'kibana/public'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, @@ -14,6 +15,7 @@ import { ErrorLike, normalizeError, inflateResponse, + DISABLE_SEARCH_COMPRESSION, } from '../../common'; import { fetchStreaming, split } from '../streaming'; import { BatchedFunc, BatchItem } from './types'; @@ -51,7 +53,7 @@ export interface StreamingBatchedFunctionParams { /** * Disabled zlib compression of response chunks. */ - disabledCompression?: boolean; + getStartServices: StartServicesAccessor; } /** @@ -69,7 +71,7 @@ export const createStreamingBatchedFunction = ( fetchStreaming: fetchStreamingInjected = fetchStreaming, flushOnMaxItems = 25, maxItemAge = 10, - disabledCompression = false, + getStartServices, } = params; const [fn] = createBatchedFunction({ onCall: (payload: Payload, signal?: AbortSignal) => { @@ -120,8 +122,16 @@ export const createStreamingBatchedFunction = ( }); const batch = items.map((item) => item.payload); + const [core] = await getStartServices(); + const disabledCompression = core.uiSettings.get(DISABLE_SEARCH_COMPRESSION); + const { stream } = fetchStreamingInjected({ url, + headers: disabledCompression + ? {} + : { + 'X-Encode-Chunks': 'true', + }, body: JSON.stringify({ batch }), method: 'POST', signal: abortController.signal, diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts index 655527b309faf..2a1bba1128906 100644 --- a/src/plugins/bfetch/public/plugin.ts +++ b/src/plugins/bfetch/public/plugin.ts @@ -7,8 +7,9 @@ */ import { CoreStart, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/public'; +import { StartServicesAccessor } from 'kibana/public'; import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './streaming'; -import { removeLeadingSlash, DISABLE_SEARCH_COMPRESSION } from '../common'; +import { removeLeadingSlash } from '../common'; import { createStreamingBatchedFunction, StreamingBatchedFunctionParams, @@ -40,7 +41,6 @@ export class BfetchPublicPlugin BfetchPublicStartDependencies > { private contract!: BfetchPublicContract; - private disabledCompression!: boolean; constructor(private readonly initializerContext: PluginInitializerContext) {} @@ -49,7 +49,7 @@ export class BfetchPublicPlugin const basePath = core.http.basePath.get(); const fetchStreaming = this.fetchStreaming(version, basePath); - const batchedFunction = this.batchedFunction(fetchStreaming); + const batchedFunction = this.batchedFunction(fetchStreaming, core.getStartServices); this.contract = { fetchStreaming, @@ -60,7 +60,6 @@ export class BfetchPublicPlugin } public start(core: CoreStart, plugins: BfetchPublicStartDependencies): BfetchPublicStart { - this.disabledCompression = core.uiSettings.get(DISABLE_SEARCH_COMPRESSION, true); return this.contract; } @@ -81,11 +80,12 @@ export class BfetchPublicPlugin }); private batchedFunction = ( - fetchStreaming: BfetchPublicContract['fetchStreaming'] + fetchStreaming: BfetchPublicContract['fetchStreaming'], + getStartServices: StartServicesAccessor ): BfetchPublicContract['batchedFunction'] => (params) => createStreamingBatchedFunction({ ...params, fetchStreaming: params.fetchStreaming || fetchStreaming, - disabledCompression: this.disabledCompression, + getStartServices, }); } diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index ac2fd597bb216..95acccf408c4f 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -142,15 +142,8 @@ export class BfetchServerPlugin public stop() {} - private async getCompressionDisabled( - request: KibanaRequest, - getStartServices: StartServicesAccessor - ) { - const [core] = await getStartServices(); - const uiSettingsClient = core.uiSettings.asScopedToClient( - core.savedObjects.getScopedClient(request) - ); - return uiSettingsClient.get(DISABLE_SEARCH_COMPRESSION); + private getCompressionDisabled(request: KibanaRequest) { + return !request.headers['x-encode-chunks']; } private addStreamingResponseRoute = ({ @@ -170,10 +163,9 @@ export class BfetchServerPlugin }, }, async (context, request, response) => { - const compressionDisabled = await this.getCompressionDisabled(request, getStartServices); - const handlerInstance = handler(request); const data = request.body; + const compressionDisabled = this.getCompressionDisabled(request); return response.ok({ headers: streamingHeaders, body: createStream(handlerInstance.getResponseStream(data), logger, compressionDisabled), @@ -194,7 +186,7 @@ export class BfetchServerPlugin response ) => { const response$ = await streamHandler(context, request); - const compressionDisabled = await this.getCompressionDisabled(request, getStartServices); + const compressionDisabled = this.getCompressionDisabled(request); return response.ok({ headers: streamingHeaders, body: createStream(response$, logger, compressionDisabled), diff --git a/src/plugins/bfetch/server/streaming/create_compressed_stream.ts b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts index 582b430be1ba2..6814ed1dd7955 100644 --- a/src/plugins/bfetch/server/streaming/create_compressed_stream.ts +++ b/src/plugins/bfetch/server/streaming/create_compressed_stream.ts @@ -14,11 +14,12 @@ import { Stream, PassThrough } from 'stream'; import { constants, deflate } from 'zlib'; const delimiter = '\n'; +const pDeflate = promisify(deflate); async function zipMessageToStream(output: PassThrough, message: string) { return new Promise(async (resolve, reject) => { try { - const gzipped = await promisify(deflate)(message, { + const gzipped = await pDeflate(message, { flush: constants.Z_SYNC_FLUSH, }); output.write(gzipped.toString('base64')); From c1c9382707d90237a1ed1917f40a8f2f3a5bf576 Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 5 May 2021 17:46:50 +0300 Subject: [PATCH 28/36] use custom headers --- .../create_streaming_batched_function.test.ts | 28 +++++++++++++++++++ .../create_streaming_batched_function.ts | 10 ++----- src/plugins/bfetch/public/plugin.ts | 13 +++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index e2ef70abe8e76..918169579f0cb 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -52,11 +52,16 @@ const setup = () => { }; describe('createStreamingBatchedFunction()', () => { + let getCompressionDisabled: any; + beforeEach(() => { + getCompressionDisabled = jest.fn().mockResolvedValue(true); + }); test('returns a function', () => { const { fetchStreaming } = setup(); const fn = createStreamingBatchedFunction({ url: '/test', fetchStreaming, + getCompressionDisabled, }); expect(typeof fn).toBe('function'); }); @@ -66,6 +71,7 @@ describe('createStreamingBatchedFunction()', () => { const fn = createStreamingBatchedFunction({ url: '/test', fetchStreaming, + getCompressionDisabled, }); const res = fn({}); expect(typeof res.then).toBe('function'); @@ -79,6 +85,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -98,6 +105,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -112,6 +120,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); fn({ foo: 'bar' }); @@ -130,6 +139,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); fn({ foo: 'bar' }); @@ -151,6 +161,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -169,6 +180,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const abortController = new AbortController(); @@ -191,6 +203,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); fn({ a: '1' }); @@ -214,6 +227,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); fn({ a: '1' }); @@ -234,6 +248,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = fn({ a: '1' }); @@ -251,6 +266,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = fn({ a: '1' }); @@ -292,6 +308,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = fn({ a: '1' }); @@ -326,6 +343,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = fn({ a: '1' }); @@ -366,6 +384,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = fn({ a: '1' }); @@ -407,6 +426,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise = fn({ a: '1' }); @@ -435,6 +455,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); @@ -487,6 +508,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const abortController = new AbortController(); @@ -516,6 +538,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const abortController = new AbortController(); @@ -554,6 +577,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); @@ -584,6 +608,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); @@ -621,6 +646,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); @@ -653,6 +679,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); @@ -689,6 +716,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, + getCompressionDisabled, }); const promise1 = of(fn({ a: '1' })); diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 4d36be72ec145..3d8863b81ad8a 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { StartServicesAccessor } from 'kibana/public'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, @@ -15,7 +14,6 @@ import { ErrorLike, normalizeError, inflateResponse, - DISABLE_SEARCH_COMPRESSION, } from '../../common'; import { fetchStreaming, split } from '../streaming'; import { BatchedFunc, BatchItem } from './types'; @@ -53,7 +51,7 @@ export interface StreamingBatchedFunctionParams { /** * Disabled zlib compression of response chunks. */ - getStartServices: StartServicesAccessor; + getCompressionDisabled: () => Promise; } /** @@ -71,7 +69,7 @@ export const createStreamingBatchedFunction = ( fetchStreaming: fetchStreamingInjected = fetchStreaming, flushOnMaxItems = 25, maxItemAge = 10, - getStartServices, + getCompressionDisabled, } = params; const [fn] = createBatchedFunction({ onCall: (payload: Payload, signal?: AbortSignal) => { @@ -121,9 +119,7 @@ export const createStreamingBatchedFunction = ( abortController.abort(); }); const batch = items.map((item) => item.payload); - - const [core] = await getStartServices(); - const disabledCompression = core.uiSettings.get(DISABLE_SEARCH_COMPRESSION); + const disabledCompression = await getCompressionDisabled(); const { stream } = fetchStreamingInjected({ url, diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts index 2a1bba1128906..266c506441f1d 100644 --- a/src/plugins/bfetch/public/plugin.ts +++ b/src/plugins/bfetch/public/plugin.ts @@ -7,9 +7,8 @@ */ import { CoreStart, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/public'; -import { StartServicesAccessor } from 'kibana/public'; import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './streaming'; -import { removeLeadingSlash } from '../common'; +import { DISABLE_SEARCH_COMPRESSION, removeLeadingSlash } from '../common'; import { createStreamingBatchedFunction, StreamingBatchedFunctionParams, @@ -49,7 +48,11 @@ export class BfetchPublicPlugin const basePath = core.http.basePath.get(); const fetchStreaming = this.fetchStreaming(version, basePath); - const batchedFunction = this.batchedFunction(fetchStreaming, core.getStartServices); + const getCompressionDisabled = async () => { + const [coreStart] = await core.getStartServices(); + return coreStart.uiSettings.get(DISABLE_SEARCH_COMPRESSION); + }; + const batchedFunction = this.batchedFunction(fetchStreaming, getCompressionDisabled); this.contract = { fetchStreaming, @@ -81,11 +84,11 @@ export class BfetchPublicPlugin private batchedFunction = ( fetchStreaming: BfetchPublicContract['fetchStreaming'], - getStartServices: StartServicesAccessor + getCompressionDisabled: () => Promise ): BfetchPublicContract['batchedFunction'] => (params) => createStreamingBatchedFunction({ ...params, fetchStreaming: params.fetchStreaming || fetchStreaming, - getStartServices, + getCompressionDisabled, }); } From b450f233d0fea22e0c60a9ea647662c6b1743bea Mon Sep 17 00:00:00 2001 From: Liza K Date: Wed, 5 May 2021 22:41:10 +0300 Subject: [PATCH 29/36] Update jest --- .../common/compress/deflate_response.ts | 22 ---- src/plugins/bfetch/common/compress/index.ts | 1 - .../create_streaming_batched_function.test.ts | 106 ++++++++++-------- src/plugins/bfetch/server/plugin.ts | 1 - 4 files changed, 58 insertions(+), 72 deletions(-) delete mode 100644 src/plugins/bfetch/common/compress/deflate_response.ts diff --git a/src/plugins/bfetch/common/compress/deflate_response.ts b/src/plugins/bfetch/common/compress/deflate_response.ts deleted file mode 100644 index ab777c3f71319..0000000000000 --- a/src/plugins/bfetch/common/compress/deflate_response.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { zlibSync, strToU8 } from 'fflate'; - -const ENCODE_THRESHOLD = 1400; - -export function deflateResponse(resp: T, compressed?: boolean) { - const strMessage = JSON.stringify(resp); - compressed = compressed ?? strMessage.length > ENCODE_THRESHOLD; - return JSON.stringify({ - compressed, - payload: compressed - ? Buffer.from(zlibSync(strToU8(strMessage), {})).toString('base64') - : strMessage, - }); -} diff --git a/src/plugins/bfetch/common/compress/index.ts b/src/plugins/bfetch/common/compress/index.ts index 0325fed343aaa..d912343fb5ee9 100644 --- a/src/plugins/bfetch/common/compress/index.ts +++ b/src/plugins/bfetch/common/compress/index.ts @@ -6,5 +6,4 @@ * Side Public License, v 1. */ -export { deflateResponse } from './deflate_response'; export { inflateResponse } from './inflate_response'; diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index 918169579f0cb..29504f5b9c537 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -10,10 +10,15 @@ import { createStreamingBatchedFunction } from './create_streaming_batched_funct import { fetchStreaming as fetchStreamingReal } from '../streaming/fetch_streaming'; import { AbortError, defer, of } from '../../../kibana_utils/public'; import { Subject } from 'rxjs'; -import { deflateResponse } from '../../common'; +import { promisify } from 'util'; +import { deflate } from 'zlib'; +const pDeflate = promisify(deflate); -const formatResponse = (resp: any, compressed: boolean = false) => { - return deflateResponse(resp, compressed) + '\n'; +const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); + +const compressResponse = async (resp: any) => { + const gzipped = await pDeflate(JSON.stringify(resp)); + return gzipped.toString('base64') + '\n'; }; const getPromiseState = (promise: Promise): Promise<'resolved' | 'rejected' | 'pending'> => @@ -166,10 +171,13 @@ describe('createStreamingBatchedFunction()', () => { expect(fetchStreaming).toHaveBeenCalledTimes(0); fn({ foo: 'bar' }); + await flushPromises(); expect(fetchStreaming).toHaveBeenCalledTimes(0); fn({ baz: 'quix' }); + await flushPromises(); expect(fetchStreaming).toHaveBeenCalledTimes(0); fn({ full: 'yep' }); + await flushPromises(); expect(fetchStreaming).toHaveBeenCalledTimes(1); }); @@ -209,6 +217,7 @@ describe('createStreamingBatchedFunction()', () => { fn({ a: '1' }); fn({ b: '2' }); fn({ c: '3' }); + await flushPromises(); expect(fetchStreaming.mock.calls[0][0]).toMatchObject({ url: '/test', @@ -233,8 +242,10 @@ describe('createStreamingBatchedFunction()', () => { fn({ a: '1' }); fn({ b: '2' }); fn({ c: '3' }); + await flushPromises(); expect(fetchStreaming).toHaveBeenCalledTimes(1); fn({ d: '4' }); + await flushPromises(); await new Promise((r) => setTimeout(r, 6)); expect(fetchStreaming).toHaveBeenCalledTimes(2); }); @@ -269,6 +280,8 @@ describe('createStreamingBatchedFunction()', () => { getCompressionDisabled, }); + await flushPromises(); + const promise1 = fn({ a: '1' }); const promise2 = fn({ b: '2' }); const promise3 = fn({ c: '3' }); @@ -279,10 +292,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise3)).toBe(true); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { foo: 'bar' }, - }) + }) + '\n' ); expect(await isPending(promise1)).toBe(true); @@ -290,10 +303,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise3)).toBe(true); stream.next( - formatResponse({ + JSON.stringify({ id: 0, result: { foo: 'bar 2' }, - }) + }) + '\n' ); expect(await isPending(promise1)).toBe(false); @@ -317,16 +330,16 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { foo: 'bar' }, - }) + }) + '\n' ); stream.next( - formatResponse({ + JSON.stringify({ id: 2, result: { foo: 'bar 2' }, - }) + }) + '\n' ); expect(await isPending(promise1)).toBe(true); @@ -343,31 +356,27 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + getCompressionDisabled: jest.fn().mockResolvedValue(false), }); + await flushPromises(); + const promise1 = fn({ a: '1' }); const promise2 = fn({ b: '2' }); const promise3 = fn({ c: '3' }); await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse( - { - id: 1, - result: { foo: 'bar' }, - }, - true - ) + await compressResponse({ + id: 1, + result: { foo: 'bar' }, + }) ); stream.next( - formatResponse( - { - id: 2, - result: { foo: 'bar 2' }, - }, - true - ) + await compressResponse({ + id: 2, + result: { foo: 'bar 2' }, + }) ); expect(await isPending(promise1)).toBe(true); @@ -393,22 +402,22 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 0, result: false, - }) + }) + '\n' ); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: 0, - }) + }) + '\n' ); stream.next( - formatResponse({ + JSON.stringify({ id: 2, result: '', - }) + }) + '\n' ); expect(await isPending(promise1)).toBe(false); @@ -435,10 +444,10 @@ describe('createStreamingBatchedFunction()', () => { expect(await isPending(promise)).toBe(true); stream.next( - formatResponse({ + JSON.stringify({ id: 0, error: { message: 'oops' }, - }) + }) + '\n' ); expect(await isPending(promise)).toBe(false); @@ -465,28 +474,28 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 2, result: { b: '3' }, - }) + }) + '\n' ); await new Promise((r) => setTimeout(r, 1)); stream.next( - formatResponse({ + JSON.stringify({ id: 1, error: { b: '2' }, - }) + }) + '\n' ); await new Promise((r) => setTimeout(r, 1)); stream.next( - formatResponse({ + JSON.stringify({ id: 0, result: { b: '1' }, - }) + }) + '\n' ); await new Promise((r) => setTimeout(r, 1)); @@ -556,10 +565,10 @@ describe('createStreamingBatchedFunction()', () => { expect(error).toBeInstanceOf(AbortError); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { b: '2' }, - }) + }) + '\n' ); await new Promise((r) => setTimeout(r, 1)); @@ -617,10 +626,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { b: '1' }, - }) + }) + '\n' ); stream.complete(); @@ -688,10 +697,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { b: '1' }, - }) + }) + '\n' ); stream.error('oops'); @@ -718,6 +727,7 @@ describe('createStreamingBatchedFunction()', () => { flushOnMaxItems: 3, getCompressionDisabled, }); + await flushPromises(); const promise1 = of(fn({ a: '1' })); const promise2 = of(fn({ a: '2' })); @@ -725,10 +735,10 @@ describe('createStreamingBatchedFunction()', () => { await new Promise((r) => setTimeout(r, 6)); stream.next( - formatResponse({ + JSON.stringify({ id: 1, result: { b: '1' }, - }) + }) + '\n' ); stream.next('Not a JSON\n'); diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index 95acccf408c4f..18ddd0aa2b01b 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -31,7 +31,6 @@ import { import { StreamingRequestHandler } from './types'; import { createStream } from './streaming'; import { getUiSettings } from './ui_settings'; -import { DISABLE_SEARCH_COMPRESSION } from '../common'; // eslint-disable-next-line export interface BfetchServerSetupDependencies {} From b27d1513595ab4e7658afa3fc5dbbf257b1df536 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 6 May 2021 12:11:32 +0300 Subject: [PATCH 30/36] fix tests --- .../create_streaming_batched_function.test.ts | 24 +++++++++++++++++++ .../create_streaming_batched_function.ts | 4 ++-- .../server/collectors/management/schema.ts | 4 ++++ .../server/collectors/management/types.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 6 +++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index 29504f5b9c537..493e14f04a907 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -386,6 +386,30 @@ describe('createStreamingBatchedFunction()', () => { expect(await promise3).toEqual({ foo: 'bar 2' }); }); + test('treats responses as compressed by defaut', async () => { + const { fetchStreaming, stream } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + flushOnMaxItems: 1, + fetchStreaming, + }); + + await flushPromises(); + + const promise1 = fn({ a: '1' }); + await new Promise((r) => setTimeout(r, 6)); + + stream.next( + await compressResponse({ + id: 0, + result: { foo: 'bar' }, + }) + ); + + expect(await isPending(promise1)).toBe(false); + expect(await promise1).toEqual({ foo: 'bar' }); + }); + test('resolves falsy results', async () => { const { fetchStreaming, stream } = setup(); const fn = createStreamingBatchedFunction({ diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 3d8863b81ad8a..df7910abf4e46 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -51,7 +51,7 @@ export interface StreamingBatchedFunctionParams { /** * Disabled zlib compression of response chunks. */ - getCompressionDisabled: () => Promise; + getCompressionDisabled?: () => Promise; } /** @@ -69,7 +69,7 @@ export const createStreamingBatchedFunction = ( fetchStreaming: fetchStreamingInjected = fetchStreaming, flushOnMaxItems = 25, maxItemAge = 10, - getCompressionDisabled, + getCompressionDisabled = () => Promise.resolve(false), } = params; const [fn] = createBatchedFunction({ onCall: (payload: Payload, signal?: AbortSignal) => { diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 06d1cd290ffd5..0d19fe7d5d174 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -388,6 +388,10 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'long', _meta: { description: 'Non-default value of setting.' }, }, + 'bfetch:disableCompression': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, 'visualization:visualize:legacyChartsLibrary': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index dfbe6bd3e0485..722d37154e4b5 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -22,6 +22,7 @@ export interface UsageStats { /** * non-sensitive settings */ + 'bfetch:disableCompression': boolean; 'autocomplete:useTimeRange': boolean; 'search:timeout': number; 'visualization:visualize:legacyChartsLibrary': boolean; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 7cd66dc8eef30..cf6af153b12e8 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -8265,6 +8265,12 @@ "description": "Non-default value of setting." } }, + "bfetch:disableCompression": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } + }, "visualization:visualize:legacyChartsLibrary": { "type": "boolean", "_meta": { From 611df93efaf84e846cf594e62c3425d27868d449 Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 6 May 2021 18:17:15 +0300 Subject: [PATCH 31/36] code review, baby! --- src/plugins/bfetch/common/constants.ts | 2 +- src/plugins/bfetch/common/index.ts | 1 - .../create_streaming_batched_function.test.ts | 117 +++++------------- .../create_streaming_batched_function.ts | 21 ++-- .../compress => public/batching}/index.ts | 5 +- src/plugins/bfetch/public/plugin.ts | 38 +++--- .../public/streaming/fetch_streaming.test.ts | 100 +++++++++++++++ .../public/streaming/fetch_streaming.ts | 41 ++++-- src/plugins/bfetch/public/streaming/index.ts | 1 + .../streaming}/inflate_response.ts | 17 +-- src/plugins/bfetch/server/plugin.ts | 2 +- src/plugins/bfetch/server/ui_settings.ts | 14 +-- test/api_integration/apis/search/bsearch.ts | 2 +- 13 files changed, 210 insertions(+), 151 deletions(-) rename src/plugins/bfetch/{common/compress => public/batching}/index.ts (74%) rename src/plugins/bfetch/{common/compress => public/streaming}/inflate_response.ts (50%) diff --git a/src/plugins/bfetch/common/constants.ts b/src/plugins/bfetch/common/constants.ts index 711f08bb06f83..bc72ad244d9be 100644 --- a/src/plugins/bfetch/common/constants.ts +++ b/src/plugins/bfetch/common/constants.ts @@ -6,4 +6,4 @@ * Side Public License, v 1. */ -export const DISABLE_SEARCH_COMPRESSION = 'bfetch:disableCompression'; +export const DISABLE_BFETCH_COMPRESSION = 'bfetch:disableCompression'; diff --git a/src/plugins/bfetch/common/index.ts b/src/plugins/bfetch/common/index.ts index cee47e14d06a6..9bf326eb4b6e5 100644 --- a/src/plugins/bfetch/common/index.ts +++ b/src/plugins/bfetch/common/index.ts @@ -10,5 +10,4 @@ export * from './util'; export * from './streaming'; export * from './buffer'; export * from './batch'; -export * from './compress'; export * from './constants'; diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index 493e14f04a907..01cebcb15963b 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -9,18 +9,10 @@ import { createStreamingBatchedFunction } from './create_streaming_batched_function'; import { fetchStreaming as fetchStreamingReal } from '../streaming/fetch_streaming'; import { AbortError, defer, of } from '../../../kibana_utils/public'; -import { Subject } from 'rxjs'; -import { promisify } from 'util'; -import { deflate } from 'zlib'; -const pDeflate = promisify(deflate); +import { Subject, of as rxof } from 'rxjs'; const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); -const compressResponse = async (resp: any) => { - const gzipped = await pDeflate(JSON.stringify(resp)); - return gzipped.toString('base64') + '\n'; -}; - const getPromiseState = (promise: Promise): Promise<'resolved' | 'rejected' | 'pending'> => Promise.race<'resolved' | 'rejected' | 'pending'>([ new Promise((resolve) => @@ -57,16 +49,12 @@ const setup = () => { }; describe('createStreamingBatchedFunction()', () => { - let getCompressionDisabled: any; - beforeEach(() => { - getCompressionDisabled = jest.fn().mockResolvedValue(true); - }); test('returns a function', () => { const { fetchStreaming } = setup(); const fn = createStreamingBatchedFunction({ url: '/test', fetchStreaming, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); expect(typeof fn).toBe('function'); }); @@ -76,7 +64,7 @@ describe('createStreamingBatchedFunction()', () => { const fn = createStreamingBatchedFunction({ url: '/test', fetchStreaming, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const res = fn({}); expect(typeof res.then).toBe('function'); @@ -90,7 +78,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -110,7 +98,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -125,7 +113,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); fn({ foo: 'bar' }); @@ -144,7 +132,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); fn({ foo: 'bar' }); @@ -166,7 +154,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); expect(fetchStreaming).toHaveBeenCalledTimes(0); @@ -188,7 +176,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const abortController = new AbortController(); @@ -211,7 +199,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); fn({ a: '1' }); @@ -236,7 +224,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); fn({ a: '1' }); @@ -259,7 +247,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = fn({ a: '1' }); @@ -277,7 +265,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); await flushPromises(); @@ -321,7 +309,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = fn({ a: '1' }); @@ -349,65 +337,18 @@ describe('createStreamingBatchedFunction()', () => { expect(await promise3).toEqual({ foo: 'bar 2' }); }); - test('handles compressed chunks', async () => { - const { fetchStreaming, stream } = setup(); - const fn = createStreamingBatchedFunction({ - url: '/test', - fetchStreaming, - maxItemAge: 5, - flushOnMaxItems: 3, - getCompressionDisabled: jest.fn().mockResolvedValue(false), - }); - - await flushPromises(); - - const promise1 = fn({ a: '1' }); - const promise2 = fn({ b: '2' }); - const promise3 = fn({ c: '3' }); - await new Promise((r) => setTimeout(r, 6)); - - stream.next( - await compressResponse({ - id: 1, - result: { foo: 'bar' }, - }) - ); - stream.next( - await compressResponse({ - id: 2, - result: { foo: 'bar 2' }, - }) - ); - - expect(await isPending(promise1)).toBe(true); - expect(await isPending(promise2)).toBe(false); - expect(await isPending(promise3)).toBe(false); - expect(await promise2).toEqual({ foo: 'bar' }); - expect(await promise3).toEqual({ foo: 'bar 2' }); - }); - - test('treats responses as compressed by defaut', async () => { - const { fetchStreaming, stream } = setup(); + test('compression is false by default', async () => { + const { fetchStreaming } = setup(); const fn = createStreamingBatchedFunction({ url: '/test', flushOnMaxItems: 1, fetchStreaming, }); - await flushPromises(); - - const promise1 = fn({ a: '1' }); - await new Promise((r) => setTimeout(r, 6)); - - stream.next( - await compressResponse({ - id: 0, - result: { foo: 'bar' }, - }) - ); + fn({ a: '1' }); - expect(await isPending(promise1)).toBe(false); - expect(await promise1).toEqual({ foo: 'bar' }); + const dontCompress = await fetchStreaming.mock.calls[0][0].compressionDisabled$.toPromise(); + expect(dontCompress).toBe(false); }); test('resolves falsy results', async () => { @@ -417,7 +358,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = fn({ a: '1' }); @@ -459,7 +400,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise = fn({ a: '1' }); @@ -488,7 +429,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = of(fn({ a: '1' })); @@ -541,7 +482,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const abortController = new AbortController(); @@ -571,7 +512,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const abortController = new AbortController(); @@ -610,7 +551,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = of(fn({ a: '1' })); @@ -641,7 +582,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = of(fn({ a: '1' })); @@ -679,7 +620,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = of(fn({ a: '1' })); @@ -712,7 +653,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); const promise1 = of(fn({ a: '1' })); @@ -749,7 +690,7 @@ describe('createStreamingBatchedFunction()', () => { fetchStreaming, maxItemAge: 5, flushOnMaxItems: 3, - getCompressionDisabled, + compressionDisabled$: rxof(true), }); await flushPromises(); diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index df7910abf4e46..d5f955f517d13 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { Observable, of } from 'rxjs'; import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, @@ -13,9 +14,8 @@ import { createBatchedFunction, ErrorLike, normalizeError, - inflateResponse, } from '../../common'; -import { fetchStreaming, split } from '../streaming'; +import { fetchStreaming } from '../streaming'; import { BatchedFunc, BatchItem } from './types'; export interface BatchedFunctionProtocolError extends ErrorLike { @@ -51,7 +51,7 @@ export interface StreamingBatchedFunctionParams { /** * Disabled zlib compression of response chunks. */ - getCompressionDisabled?: () => Promise; + compressionDisabled$?: Observable; } /** @@ -69,7 +69,7 @@ export const createStreamingBatchedFunction = ( fetchStreaming: fetchStreamingInjected = fetchStreaming, flushOnMaxItems = 25, maxItemAge = 10, - getCompressionDisabled = () => Promise.resolve(false), + compressionDisabled$ = of(false), } = params; const [fn] = createBatchedFunction({ onCall: (payload: Payload, signal?: AbortSignal) => { @@ -119,18 +119,13 @@ export const createStreamingBatchedFunction = ( abortController.abort(); }); const batch = items.map((item) => item.payload); - const disabledCompression = await getCompressionDisabled(); const { stream } = fetchStreamingInjected({ url, - headers: disabledCompression - ? {} - : { - 'X-Encode-Chunks': 'true', - }, body: JSON.stringify({ batch }), method: 'POST', signal: abortController.signal, + compressionDisabled$, }); const handleStreamError = (error: any) => { @@ -139,12 +134,10 @@ export const createStreamingBatchedFunction = ( for (const { future } of items) future.reject(normalizedError); }; - stream.pipe(split('\n')).subscribe({ + stream.subscribe({ next: (json: string) => { try { - const response = disabledCompression - ? JSON.parse(json) - : inflateResponse(json); + const response = JSON.parse(json); if (response.error) { items[response.id].future.reject(response.error); } else if (response.result !== undefined) { diff --git a/src/plugins/bfetch/common/compress/index.ts b/src/plugins/bfetch/public/batching/index.ts similarity index 74% rename from src/plugins/bfetch/common/compress/index.ts rename to src/plugins/bfetch/public/batching/index.ts index d912343fb5ee9..115fd84cbe979 100644 --- a/src/plugins/bfetch/common/compress/index.ts +++ b/src/plugins/bfetch/public/batching/index.ts @@ -6,4 +6,7 @@ * Side Public License, v 1. */ -export { inflateResponse } from './inflate_response'; +export { + createStreamingBatchedFunction, + StreamingBatchedFunctionParams, +} from './create_streaming_batched_function'; diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts index 266c506441f1d..f97a91a0e70d3 100644 --- a/src/plugins/bfetch/public/plugin.ts +++ b/src/plugins/bfetch/public/plugin.ts @@ -7,12 +7,11 @@ */ import { CoreStart, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/public'; +import { from, Observable, of } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './streaming'; -import { DISABLE_SEARCH_COMPRESSION, removeLeadingSlash } from '../common'; -import { - createStreamingBatchedFunction, - StreamingBatchedFunctionParams, -} from './batching/create_streaming_batched_function'; +import { DISABLE_BFETCH_COMPRESSION, removeLeadingSlash } from '../common'; +import { createStreamingBatchedFunction, StreamingBatchedFunctionParams } from './batching'; import { BatchedFunc } from './batching/types'; // eslint-disable-next-line @@ -43,16 +42,23 @@ export class BfetchPublicPlugin constructor(private readonly initializerContext: PluginInitializerContext) {} - public setup(core: CoreSetup, plugins: BfetchPublicSetupDependencies): BfetchPublicSetup { + public setup( + core: CoreSetup, + plugins: BfetchPublicSetupDependencies + ): BfetchPublicSetup { const { version } = this.initializerContext.env.packageInfo; const basePath = core.http.basePath.get(); - const fetchStreaming = this.fetchStreaming(version, basePath); - const getCompressionDisabled = async () => { - const [coreStart] = await core.getStartServices(); - return coreStart.uiSettings.get(DISABLE_SEARCH_COMPRESSION); - }; - const batchedFunction = this.batchedFunction(fetchStreaming, getCompressionDisabled); + const compressionDisabled$ = from(core.getStartServices()).pipe( + switchMap((deps) => { + return of(deps[0]); + }), + switchMap((coreStart) => { + return coreStart.uiSettings.get$(DISABLE_BFETCH_COMPRESSION); + }) + ); + const fetchStreaming = this.fetchStreaming(version, basePath, compressionDisabled$); + const batchedFunction = this.batchedFunction(fetchStreaming, compressionDisabled$); this.contract = { fetchStreaming, @@ -70,7 +76,8 @@ export class BfetchPublicPlugin private fetchStreaming = ( version: string, - basePath: string + basePath: string, + compressionDisabled$: Observable ): BfetchPublicSetup['fetchStreaming'] => (params) => fetchStreamingStatic({ ...params, @@ -80,15 +87,16 @@ export class BfetchPublicPlugin 'kbn-version': version, ...(params.headers || {}), }, + compressionDisabled$, }); private batchedFunction = ( fetchStreaming: BfetchPublicContract['fetchStreaming'], - getCompressionDisabled: () => Promise + compressionDisabled$: Observable ): BfetchPublicContract['batchedFunction'] => (params) => createStreamingBatchedFunction({ ...params, + compressionDisabled$, fetchStreaming: params.fetchStreaming || fetchStreaming, - getCompressionDisabled, }); } diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts index e804b3ea94227..452df25f0c2c2 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts @@ -8,6 +8,15 @@ import { fetchStreaming } from './fetch_streaming'; import { mockXMLHttpRequest } from '../test_helpers/xhr'; +import { of } from 'rxjs'; +import { promisify } from 'util'; +import { deflate } from 'zlib'; +const pDeflate = promisify(deflate); + +const compressResponse = async (resp: any) => { + const gzipped = await pDeflate(JSON.stringify(resp)); + return gzipped.toString('base64'); +}; const tick = () => new Promise((resolve) => setTimeout(resolve, 1)); @@ -21,6 +30,7 @@ test('returns XHR request', () => { setup(); const { xhr } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); expect(typeof xhr.readyState).toBe('number'); }); @@ -29,6 +39,7 @@ test('returns stream', () => { setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); expect(typeof stream.subscribe).toBe('function'); }); @@ -37,6 +48,7 @@ test('promise resolves when request completes', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); let resolved = false; @@ -65,10 +77,90 @@ test('promise resolves when request completes', async () => { expect(resolved).toBe(true); }); +test('promise resolves when compressed request completes', async () => { + const env = setup(); + const { stream } = fetchStreaming({ + url: 'http://example.com', + compressionDisabled$: of(false), + }); + + let resolved = false; + let result; + stream.toPromise().then((r) => { + resolved = true; + result = r; + }); + + await tick(); + expect(resolved).toBe(false); + + const msg = { foo: 'bar' }; + + // Whole message in a response + (env.xhr as any).responseText = `${await compressResponse(msg)}\n`; + env.xhr.onprogress!({} as any); + + await tick(); + expect(resolved).toBe(false); + + (env.xhr as any).readyState = 4; + (env.xhr as any).status = 200; + env.xhr.onreadystatechange!({} as any); + + await tick(); + expect(resolved).toBe(true); + expect(result).toStrictEqual(JSON.stringify(msg)); +}); + +test('promise resolves when compressed chunked request completes', async () => { + const env = setup(); + const { stream } = fetchStreaming({ + url: 'http://example.com', + compressionDisabled$: of(false), + }); + + let resolved = false; + let result; + stream.toPromise().then((r) => { + resolved = true; + result = r; + }); + + await tick(); + expect(resolved).toBe(false); + + const msg = { veg: 'tomato' }; + const msgToCut = await compressResponse(msg); + const part1 = msgToCut.substr(0, 3); + + // Message and a half in a response + (env.xhr as any).responseText = part1; + env.xhr.onprogress!({} as any); + + await tick(); + expect(resolved).toBe(false); + + // Half a message in a response + (env.xhr as any).responseText = `${msgToCut}\n`; + env.xhr.onprogress!({} as any); + + await tick(); + expect(resolved).toBe(false); + + (env.xhr as any).readyState = 4; + (env.xhr as any).status = 200; + env.xhr.onreadystatechange!({} as any); + + await tick(); + expect(resolved).toBe(true); + expect(result).toStrictEqual(JSON.stringify(msg)); +}); + test('streams incoming text as it comes through', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); const spy = jest.fn(); @@ -103,6 +195,7 @@ test('completes stream observable when request finishes', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); const spy = jest.fn(); @@ -127,6 +220,7 @@ test('completes stream observable when aborted', async () => { const { stream } = fetchStreaming({ url: 'http://example.com', signal: abort.signal, + compressionDisabled$: of(true), }); const spy = jest.fn(); @@ -152,6 +246,7 @@ test('promise throws when request errors', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); const spy = jest.fn(); @@ -178,6 +273,7 @@ test('stream observable errors when request errors', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); const spy = jest.fn(); @@ -210,6 +306,7 @@ test('sets custom headers', async () => { 'Content-Type': 'text/plain', Authorization: 'Bearer 123', }, + compressionDisabled$: of(true), }); expect(env.xhr.setRequestHeader).toHaveBeenCalledWith('Content-Type', 'text/plain'); @@ -223,6 +320,7 @@ test('uses credentials', async () => { fetchStreaming({ url: 'http://example.com', + compressionDisabled$: of(true), }); expect(env.xhr.withCredentials).toBe(true); @@ -238,6 +336,7 @@ test('opens XHR request and sends specified body', async () => { url: 'http://elastic.co', method: 'GET', body: 'foobar', + compressionDisabled$: of(true), }); expect(env.xhr.open).toHaveBeenCalledTimes(1); @@ -250,6 +349,7 @@ test('uses POST request method by default', async () => { const env = setup(); fetchStreaming({ url: 'http://elastic.co', + compressionDisabled$: of(true), }); expect(env.xhr.open).toHaveBeenCalledWith('POST', 'http://elastic.co'); }); diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.ts index d68e4d01b44f5..13a3571a4d4c3 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.ts @@ -6,7 +6,11 @@ * Side Public License, v 1. */ +import { Observable } from 'rxjs'; +import { map, share, switchMap } from 'rxjs/operators'; +import { inflateResponse } from '.'; import { fromStreamingXhr } from './from_streaming_xhr'; +import { split } from './split'; export interface FetchStreamingParams { url: string; @@ -14,6 +18,7 @@ export interface FetchStreamingParams { method?: 'GET' | 'POST'; body?: string; signal?: AbortSignal; + compressionDisabled$: Observable; } /** @@ -26,23 +31,41 @@ export function fetchStreaming({ method = 'POST', body = '', signal, + compressionDisabled$, }: FetchStreamingParams) { const xhr = new window.XMLHttpRequest(); - // Begin the request - xhr.open(method, url); - xhr.withCredentials = true; + const msgStream = compressionDisabled$.pipe( + switchMap((compressionDisabled) => { + // Begin the request + xhr.open(method, url); + xhr.withCredentials = true; - // Set the HTTP headers - Object.entries(headers).forEach(([k, v]) => xhr.setRequestHeader(k, v)); + if (!compressionDisabled) { + headers['X-Chunk-Encoding'] = 'deflate'; + } - const stream = fromStreamingXhr(xhr, signal); + // Set the HTTP headers + Object.entries(headers).forEach(([k, v]) => xhr.setRequestHeader(k, v)); - // Send the payload to the server - xhr.send(body); + const stream = fromStreamingXhr(xhr, signal); + + // Send the payload to the server + xhr.send(body); + + // Return a stream of chunked decompressed messages + return stream.pipe( + split('\n'), + map((msg) => { + return compressionDisabled ? msg : inflateResponse(msg); + }) + ); + }), + share() + ); return { xhr, - stream, + stream: msgStream, }; } diff --git a/src/plugins/bfetch/public/streaming/index.ts b/src/plugins/bfetch/public/streaming/index.ts index afb442feffb29..545cae87aa3d6 100644 --- a/src/plugins/bfetch/public/streaming/index.ts +++ b/src/plugins/bfetch/public/streaming/index.ts @@ -9,3 +9,4 @@ export * from './split'; export * from './from_streaming_xhr'; export * from './fetch_streaming'; +export { inflateResponse } from './inflate_response'; diff --git a/src/plugins/bfetch/common/compress/inflate_response.ts b/src/plugins/bfetch/public/streaming/inflate_response.ts similarity index 50% rename from src/plugins/bfetch/common/compress/inflate_response.ts rename to src/plugins/bfetch/public/streaming/inflate_response.ts index 0590152316cef..73cb52285987c 100644 --- a/src/plugins/bfetch/common/compress/inflate_response.ts +++ b/src/plugins/bfetch/public/streaming/inflate_response.ts @@ -7,18 +7,9 @@ */ import { unzlibSync, strFromU8 } from 'fflate'; -import { BatchResponseItem, ErrorLike } from '../../common'; -export function inflateResponse( - response: string -): BatchResponseItem { - try { - const buff = Buffer.from(response, 'base64'); - - const unzip = unzlibSync(buff); - const inflatedRes = strFromU8(unzip); - return JSON.parse(inflatedRes); - } catch (e) { - return JSON.parse(response); - } +export function inflateResponse(response: string) { + const buff = Buffer.from(response, 'base64'); + const unzip = unzlibSync(buff); + return strFromU8(unzip); } diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index 18ddd0aa2b01b..6a878241ed719 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -142,7 +142,7 @@ export class BfetchServerPlugin public stop() {} private getCompressionDisabled(request: KibanaRequest) { - return !request.headers['x-encode-chunks']; + return !request.headers['x-chunk-encoding']; } private addStreamingResponseRoute = ({ diff --git a/src/plugins/bfetch/server/ui_settings.ts b/src/plugins/bfetch/server/ui_settings.ts index 2fb9268f368d8..cf7b13a9af182 100644 --- a/src/plugins/bfetch/server/ui_settings.ts +++ b/src/plugins/bfetch/server/ui_settings.ts @@ -9,21 +9,21 @@ import { i18n } from '@kbn/i18n'; import { UiSettingsParams } from 'src/core/server'; import { schema } from '@kbn/config-schema'; -import { DISABLE_SEARCH_COMPRESSION } from '../common'; +import { DISABLE_BFETCH_COMPRESSION } from '../common'; export function getUiSettings(): Record> { return { - [DISABLE_SEARCH_COMPRESSION]: { - name: i18n.translate('bfetch.disableSearchCompression', { - defaultMessage: 'Disable Search Compression', + [DISABLE_BFETCH_COMPRESSION]: { + name: i18n.translate('bfetch.disableBfetchCompression', { + defaultMessage: 'Disable Batch Compression', }), value: false, - description: i18n.translate('bfetch.disableSearchCompressionDesc', { + description: i18n.translate('bfetch.disableBfetchCompressionDesc', { defaultMessage: - 'Disable search compression. This allows you debug individual search requests, but increases response size.', + 'Disable batch compression. This allows you to debug individual requests, but increases response size.', }), schema: schema.boolean(), - category: ['search'], + category: [], }, }; } diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index fea942779d60d..9f94c1b3d35e9 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import request from 'superagent'; -import { inflateResponse } from '../../../../src/plugins/bfetch/common'; +import { inflateResponse } from '../../../../src/plugins/bfetch/public/streaming'; import { FtrProviderContext } from '../../ftr_provider_context'; import { painlessErrReq } from './painless_err_req'; import { verifyErrorResponse } from './verify_error'; From a06d709ebd0bbdac9066fb53fb0cd1cb2ae76ebe Mon Sep 17 00:00:00 2001 From: Liza K Date: Thu, 6 May 2021 18:38:18 +0300 Subject: [PATCH 32/36] integration --- test/api_integration/apis/search/bsearch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index 9f94c1b3d35e9..4b7f240e2a87f 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -18,7 +18,7 @@ function parseBfetchResponse(resp: request.Response) { .trim() .split('\n') .map((item) => { - return inflateResponse(item); + return JSON.parse(inflateResponse(item)); }); } From 9d21b3aea5abb3455c1693309fab2d58372ddfa4 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 9 May 2021 18:33:19 +0300 Subject: [PATCH 33/36] tests --- .../public/streaming/fetch_streaming.test.ts | 12 +++- .../public/streaming/fetch_streaming.ts | 14 +++- src/plugins/bfetch/server/plugin.ts | 2 +- test/api_integration/apis/search/bsearch.ts | 70 ++++++++++++++----- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts index 452df25f0c2c2..a5d066f6d9a24 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts @@ -156,7 +156,7 @@ test('promise resolves when compressed chunked request completes', async () => { expect(result).toStrictEqual(JSON.stringify(msg)); }); -test('streams incoming text as it comes through', async () => { +test('streams incoming text as it comes through, according to separators', async () => { const env = setup(); const { stream } = fetchStreaming({ url: 'http://example.com', @@ -172,16 +172,22 @@ test('streams incoming text as it comes through', async () => { (env.xhr as any).responseText = 'foo'; env.xhr.onprogress!({} as any); + await tick(); + expect(spy).toHaveBeenCalledTimes(0); + + (env.xhr as any).responseText = 'foo\nbar'; + env.xhr.onprogress!({} as any); + await tick(); expect(spy).toHaveBeenCalledTimes(1); expect(spy).toHaveBeenCalledWith('foo'); - (env.xhr as any).responseText = 'foo\nbar'; + (env.xhr as any).responseText = 'foo\nbar\n'; env.xhr.onprogress!({} as any); await tick(); expect(spy).toHaveBeenCalledTimes(2); - expect(spy).toHaveBeenCalledWith('\nbar'); + expect(spy).toHaveBeenCalledWith('bar'); (env.xhr as any).readyState = 4; (env.xhr as any).status = 200; diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.ts index 13a3571a4d4c3..1af35ef68fb85 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { Observable } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { map, share, switchMap } from 'rxjs/operators'; import { inflateResponse } from '.'; import { fromStreamingXhr } from './from_streaming_xhr'; @@ -18,7 +18,7 @@ export interface FetchStreamingParams { method?: 'GET' | 'POST'; body?: string; signal?: AbortSignal; - compressionDisabled$: Observable; + compressionDisabled$?: Observable; } /** @@ -31,7 +31,7 @@ export function fetchStreaming({ method = 'POST', body = '', signal, - compressionDisabled$, + compressionDisabled$ = of(false), }: FetchStreamingParams) { const xhr = new window.XMLHttpRequest(); @@ -64,6 +64,14 @@ export function fetchStreaming({ share() ); + // start execution + const msgStreamSub = msgStream.subscribe({ + error: (e) => {}, + complete: () => { + msgStreamSub.unsubscribe(); + }, + }); + return { xhr, stream: msgStream, diff --git a/src/plugins/bfetch/server/plugin.ts b/src/plugins/bfetch/server/plugin.ts index 6a878241ed719..7fd46e2f6cc44 100644 --- a/src/plugins/bfetch/server/plugin.ts +++ b/src/plugins/bfetch/server/plugin.ts @@ -142,7 +142,7 @@ export class BfetchServerPlugin public stop() {} private getCompressionDisabled(request: KibanaRequest) { - return !request.headers['x-chunk-encoding']; + return request.headers['x-chunk-encoding'] !== 'deflate'; } private addStreamingResponseRoute = ({ diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index 7fdd99f2f6fac..11fb74200d7dd 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -13,12 +13,12 @@ import { FtrProviderContext } from '../../ftr_provider_context'; import { painlessErrReq } from './painless_err_req'; import { verifyErrorResponse } from './verify_error'; -function parseBfetchResponse(resp: request.Response) { +function parseBfetchResponse(resp: request.Response, compressed: boolean = false) { return resp.text .trim() .split('\n') .map((item) => { - return JSON.parse(inflateResponse(item)); + return JSON.parse(compressed ? inflateResponse(item) : item); }); } @@ -29,25 +29,28 @@ export default function ({ getService }: FtrProviderContext) { describe('bsearch', () => { describe('post', () => { it('should return 200 a single response', async () => { - const resp = await supertest.post(`/internal/bsearch`).send({ - batch: [ - { - request: { - params: { - index: '.kibana', - body: { - query: { - match_all: {}, + const resp = await supertest + .post(`/internal/bsearch`) + .set({ 'X-Chunk-Encoding': '' }) + .send({ + batch: [ + { + request: { + params: { + index: '.kibana', + body: { + query: { + match_all: {}, + }, }, }, }, + options: { + strategy: 'es', + }, }, - options: { - strategy: 'es', - }, - }, - ], - }); + ], + }); const jsonBody = parseBfetchResponse(resp); @@ -58,6 +61,39 @@ export default function ({ getService }: FtrProviderContext) { expect(jsonBody[0].result).to.have.property('rawResponse'); }); + it('should return 200 a single response from compressed', async () => { + const resp = await supertest + .post(`/internal/bsearch`) + .set({ 'X-Chunk-Encoding': 'deflate' }) + .send({ + batch: [ + { + request: { + params: { + index: '.kibana', + body: { + query: { + match_all: {}, + }, + }, + }, + }, + options: { + strategy: 'es', + }, + }, + ], + }); + + const jsonBody = parseBfetchResponse(resp, true); + + expect(resp.status).to.be(200); + expect(jsonBody[0].id).to.be(0); + expect(jsonBody[0].result.isPartial).to.be(false); + expect(jsonBody[0].result.isRunning).to.be(false); + expect(jsonBody[0].result).to.have.property('rawResponse'); + }); + it('should return a batch of successful responses', async () => { const resp = await supertest.post(`/internal/bsearch`).send({ batch: [ From a139f9e5237ce20b2f03a9f9085a6638296ec845 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 9 May 2021 19:51:49 +0300 Subject: [PATCH 34/36] limit --- packages/kbn-optimizer/limits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 63dd64f9202b3..cda13a909a16e 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -4,7 +4,7 @@ pageLoadAssetSize: apm: 64385 apmOss: 18996 beatsManagement: 188135 - bfetch: 41874 + bfetch: 46874 canvas: 1066647 charts: 195358 cloud: 21076 From b442879244bf2b2ccd08cce1c54430cc30738722 Mon Sep 17 00:00:00 2001 From: Liza K Date: Sun, 9 May 2021 20:33:46 +0300 Subject: [PATCH 35/36] limit --- packages/kbn-optimizer/limits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index cda13a909a16e..560f4427f237c 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -4,7 +4,7 @@ pageLoadAssetSize: apm: 64385 apmOss: 18996 beatsManagement: 188135 - bfetch: 46874 + bfetch: 51874 canvas: 1066647 charts: 195358 cloud: 21076 From d07b8ebfe38c0c50e2f4410a12b9c17312854fd0 Mon Sep 17 00:00:00 2001 From: Liza K Date: Mon, 31 May 2021 15:12:26 +0300 Subject: [PATCH 36/36] limit --- packages/kbn-optimizer/limits.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index c3f9b32d87b18..6ccf6269751b1 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -3,7 +3,6 @@ pageLoadAssetSize: alerting: 106936 apm: 64385 apmOss: 18996 - beatsManagement: 188135 bfetch: 51874 canvas: 1066647 charts: 195358