From 6afce3e2b7ff0dcb0cf652ebb23530ab7e9f1e46 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 20 Mar 2023 21:47:17 -0500 Subject: [PATCH 1/5] remove partial index pattern validation --- .../fetcher/index_patterns_fetcher.test.ts | 60 +------------------ .../server/fetcher/index_patterns_fetcher.ts | 41 +------------ 2 files changed, 4 insertions(+), 97 deletions(-) diff --git a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts index 81a58c844fee4..f6f2d378fef7c 100644 --- a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts +++ b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts @@ -9,14 +9,10 @@ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { IndexPatternsFetcher } from '.'; import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; -import * as indexNotFoundException from './index_not_found_exception.json'; describe('Index Pattern Fetcher - server', () => { let indexPatterns: IndexPatternsFetcher; let esClient: ReturnType; - const emptyResponse = { - indices: [], - }; const response = { indices: ['b'], fields: [{ name: 'foo' }, { name: 'bar' }, { name: 'baz' }], @@ -27,61 +23,7 @@ describe('Index Pattern Fetcher - server', () => { esClient = elasticsearchServiceMock.createElasticsearchClient(); indexPatterns = new IndexPatternsFetcher(esClient); }); - it('Removes pattern without matching indices', async () => { - esClient.fieldCaps - .mockResponseOnce(emptyResponse as unknown as estypes.FieldCapsResponse) - .mockResponse(response as unknown as estypes.FieldCapsResponse); - // first field caps request returns empty - const result = await indexPatterns.validatePatternListActive(patternList); - expect(result).toEqual(['b', 'c']); - }); - it('Keeps matching and negating patterns', async () => { - esClient.fieldCaps - .mockResponseOnce(emptyResponse as unknown as estypes.FieldCapsResponse) - .mockResponse(response as unknown as estypes.FieldCapsResponse); - // first field caps request returns empty - const result = await indexPatterns.validatePatternListActive(['-a', 'b', 'c', 'a:-b']); - expect(result).toEqual(['-a', 'c', 'a:-b']); - }); - it('Returns all patterns when all match indices', async () => { - esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse); - indexPatterns = new IndexPatternsFetcher(esClient); - const result = await indexPatterns.validatePatternListActive(patternList); - expect(result).toEqual(patternList); - }); - it('Removes pattern when error is thrown', async () => { - class ServerError extends Error { - public body?: Record; - - constructor( - message: string, - public readonly statusCode: number, - errBody?: Record - ) { - super(message); - this.body = errBody; - } - } - - esClient.fieldCaps - .mockResponseOnce(response as unknown as estypes.FieldCapsResponse) - .mockImplementationOnce(() => { - return Promise.reject( - new ServerError('index_not_found_exception', 404, indexNotFoundException) - ); - }); - - indexPatterns = new IndexPatternsFetcher(esClient); - const result = await indexPatterns.validatePatternListActive(patternList); - expect(result).toEqual([patternList[0]]); - }); - it('When allowNoIndices is false, run validatePatternListActive', async () => { - esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse); - indexPatterns = new IndexPatternsFetcher(esClient); - await indexPatterns.getFieldsForWildcard({ pattern: patternList }); - expect(esClient.fieldCaps).toHaveBeenCalledTimes(4); - }); - it('When allowNoIndices is true, do not run validatePatternListActive', async () => { + it('calls fieldcaps once', async () => { esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse); indexPatterns = new IndexPatternsFetcher(esClient, true); await indexPatterns.getFieldsForWildcard({ pattern: patternList }); diff --git a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts index df18a5d506f17..dc73cebb8855d 100644 --- a/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts @@ -64,18 +64,13 @@ export class IndexPatternsFetcher { fields?: string[]; }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> { const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options; - const patternList = Array.isArray(pattern) ? pattern : pattern.split(','); const allowNoIndices = fieldCapsOptions ? fieldCapsOptions.allow_no_indices : this.allowNoIndices; - let patternListActive: string[] = patternList; - // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless - if (patternList.length > 1 && !allowNoIndices) { - patternListActive = await this.validatePatternListActive(patternList); - } + const fieldCapsResponse = await getFieldCapabilities({ callCluster: this.elasticsearchClient, - indices: patternListActive, + indices: pattern, metaFields, fieldCapsOptions: { allow_no_indices: allowNoIndices, @@ -84,6 +79,7 @@ export class IndexPatternsFetcher { indexFilter, fields: options.fields || ['*'], }); + if (type === 'rollup' && rollupIndex) { const rollupFields: FieldDescriptor[] = []; const capabilityCheck = getCapabilitiesForRollupIndices( @@ -114,35 +110,4 @@ export class IndexPatternsFetcher { } return fieldCapsResponse; } - - /** - * Returns an index pattern list of only those index pattern strings in the given list that return indices - * - * @param patternList string[] - * @return {Promise} - */ - async validatePatternListActive(patternList: string[]) { - const result = await Promise.all( - patternList - .map(async (index) => { - // perserve negated patterns - if (index.startsWith('-') || index.includes(':-')) { - return true; - } - const searchResponse = await this.elasticsearchClient.fieldCaps({ - index, - fields: '_id', - ignore_unavailable: true, - allow_no_indices: false, - }); - return searchResponse.indices.length > 0; - }) - .map((p) => p.catch(() => false)) - ); - return result.reduce( - (acc: string[], isValid, patternListIndex) => - isValid ? [...acc, patternList[patternListIndex]] : acc, - [] - ); - } } From 910c79543a2d46f383833d025ade1fe8e8bbc07a Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 21 Mar 2023 10:18:34 -0500 Subject: [PATCH 2/5] fix hybrid index pattern creation --- src/plugins/data_view_editor/public/data_view_editor_service.ts | 1 + x-pack/test/functional/config.base.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_view_editor/public/data_view_editor_service.ts b/src/plugins/data_view_editor/public/data_view_editor_service.ts index 798e339b7ee3f..8d709ca30fe32 100644 --- a/src/plugins/data_view_editor/public/data_view_editor_service.ts +++ b/src/plugins/data_view_editor/public/data_view_editor_service.ts @@ -334,6 +334,7 @@ export class DataViewEditorService { if (this.type === INDEX_PATTERN_TYPE.ROLLUP) { getFieldsOptions.type = INDEX_PATTERN_TYPE.ROLLUP; getFieldsOptions.rollupIndex = currentState.rollupIndexName || ''; + getFieldsOptions.allowNoIndex = true; } let timestampFieldOptions: TimestampOption[] = []; diff --git a/x-pack/test/functional/config.base.js b/x-pack/test/functional/config.base.js index 04c62d5e1204b..b6ac951f2a45c 100644 --- a/x-pack/test/functional/config.base.js +++ b/x-pack/test/functional/config.base.js @@ -461,7 +461,7 @@ export default async function ({ readConfigFile }) { elasticsearch: { indices: [ { - names: ['rollup-*'], + names: ['rollup-*', 'regular-index*'], privileges: ['read', 'view_index_metadata'], }, ], From 885f2b179276510754658b79ccb5a50ca409ccbf Mon Sep 17 00:00:00 2001 From: shahzad31 Date: Wed, 22 Mar 2023 10:10:28 +0100 Subject: [PATCH 3/5] fix tests --- .../utils/observability_data_views/observability_data_views.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts index 0c9ad40784ec8..c7b3d1b78b363 100644 --- a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts +++ b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts @@ -132,6 +132,7 @@ export class ObservabilityDataViews { timeFieldName: '@timestamp', fieldFormats: this.getFieldFormats(app), name: DataTypesLabels[app], + allowNoIndex: true, }, false, false @@ -162,6 +163,7 @@ export class ObservabilityDataViews { timeFieldName: '@timestamp', fieldFormats: this.getFieldFormats(app), name: DataTypesLabels[app], + allowNoIndex: true, }); } // we want to make sure field formats remain same From 12f144f68f7d71f01fa19eae5a35aa77a7b7d5e7 Mon Sep 17 00:00:00 2001 From: shahzad31 Date: Wed, 22 Mar 2023 13:17:50 +0100 Subject: [PATCH 4/5] update test --- .../observability_data_views/observability_data_views.test.ts | 1 + .../observability_data_views/observability_data_views.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.test.ts b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.test.ts index 4b9b904b73fc4..26ceafb80cf5c 100644 --- a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.test.ts +++ b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.test.ts @@ -106,6 +106,7 @@ describe('ObservabilityDataViews', function () { timeFieldName: '@timestamp', title: 'trace-*,apm-*', name: 'User experience (RUM)', + allowNoIndex: true, }); expect(dataViews?.createAndSave).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts index c7b3d1b78b363..75851f87fa8d6 100644 --- a/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts +++ b/x-pack/plugins/observability/public/utils/observability_data_views/observability_data_views.ts @@ -138,6 +138,10 @@ export class ObservabilityDataViews { false ); + if (dataView.matchedIndices.length === 0) { + return; + } + if (runtimeFields !== null) { runtimeFields.forEach(({ name, field }) => { dataView.addRuntimeField(name, field); From b42a522a2e130b5e257fce0d8d42d5194eb21590 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 24 Mar 2023 10:20:38 -0500 Subject: [PATCH 5/5] add integration api test --- .../apis/data_views/fields_for_wildcard_route/response.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/api_integration/apis/data_views/fields_for_wildcard_route/response.js b/test/api_integration/apis/data_views/fields_for_wildcard_route/response.js index 9a7968d2c2531..e120a43875c84 100644 --- a/test/api_integration/apis/data_views/fields_for_wildcard_route/response.js +++ b/test/api_integration/apis/data_views/fields_for_wildcard_route/response.js @@ -201,6 +201,14 @@ export default function ({ getService }) { indices: ['basic_index'], }); }); + + it('returns 404 when neither exists', async () => { + await supertest + .get('/api/index_patterns/_fields_for_wildcard') + .query({ pattern: 'bad_index,bad_index_2' }) + .expect(404); + }); + it('returns 404 when no patterns exist', async () => { await supertest .get('/api/index_patterns/_fields_for_wildcard')