From 589a9254c4636a01d8732cc66b52e3459d84fc3b Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Tue, 16 Sep 2025 10:09:44 -0400 Subject: [PATCH] [Investigations][Bug] - Check for empty dataView (#235144) ## Summary This PR fixes an issue with the alert page filtering when the below config is enabled: image When enabled, the config looks to make sure that searches are only done against index patterns that are mapped to the given dataView. When introducing the code to migrate to our new dataView picker [here](https://github.com/elastic/kibana/blob/9659a525327b2e46478f45d03ce39103848361cc/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.ts#L231) in the following PR https://github.com/elastic/kibana/pull/225726, a check was done to only apply the new DataView when it was provided. To fix a separate issue regarding flashing of the alerts page, this following [initial dataView](https://github.com/elastic/kibana/blob/9659a525327b2e46478f45d03ce39103848361cc/x-pack/solutions/security/plugins/security_solution/public/data_view_manager/hooks/use_data_view.ts#L45) was introduced with this pr: https://github.com/elastic/kibana/pull/225675 In short, the dataView object was always defined, even if it was just an initial dataView leading to the fields being queried against not being mapped. The necessary checks are added in this PR ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios (cherry picked from commit 128528cbfe123c5f0234824e5834755cab58b0c4) --- .../public/common/lib/kuery/index.test.ts | 260 ++++++++++++------ .../public/common/lib/kuery/index.ts | 5 +- 2 files changed, 172 insertions(+), 93 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.test.ts b/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.test.ts index 22f32fc1e10a1..e93c5cf543be5 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.test.ts @@ -58,115 +58,117 @@ describe('convertToBuildEsQuery', () => { dateFormatTZ: 'Browser', }; - it('should, by default, build a query where the `nested` fields syntax includes the `"ignore_unmapped":true` option', () => { - const [converted, _] = convertToBuildEsQuery({ - config, - dataView: createStubDataView({ spec: {} }), - queries: queryWithNestedFields, - dataViewSpec: mockDataViewSpec, - filters, - }); - - expect(JSON.parse(converted ?? '')).to.eql({ - bool: { - must: [], - filter: [ - { - bool: { - filter: [ - { - bool: { - filter: [ - { - // ✅ Nested fields are converted to use the `nested` query syntax - nested: { - path: 'threat.enrichments', - query: { - bool: { - should: [ - { - match: { - 'threat.enrichments.matched.atomic': - 'a4f87cbcd2a4241da77b6bf0c5d9e8553fec991f', - }, + const expectedConverted = { + bool: { + must: [], + filter: [ + { + bool: { + filter: [ + { + bool: { + filter: [ + { + // ✅ Nested fields are converted to use the `nested` query syntax + nested: { + path: 'threat.enrichments', + query: { + bool: { + should: [ + { + match: { + 'threat.enrichments.matched.atomic': + 'a4f87cbcd2a4241da77b6bf0c5d9e8553fec991f', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - score_mode: 'none', - // ✅ The `nested` query syntax includes the `ignore_unmapped` option - ignore_unmapped: true, }, + score_mode: 'none', + // ✅ The `nested` query syntax includes the `ignore_unmapped` option + ignore_unmapped: true, }, - { - nested: { - path: 'threat.enrichments', - query: { - bool: { - should: [ - { - match: { - 'threat.enrichments.matched.type': 'indicator_match_rule', - }, + }, + { + nested: { + path: 'threat.enrichments', + query: { + bool: { + should: [ + { + match: { + 'threat.enrichments.matched.type': 'indicator_match_rule', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - score_mode: 'none', - ignore_unmapped: true, }, + score_mode: 'none', + ignore_unmapped: true, }, - { - nested: { - path: 'threat.enrichments', - query: { - bool: { - should: [ - { - match: { - 'threat.enrichments.matched.field': 'file.hash.md5', - }, + }, + { + nested: { + path: 'threat.enrichments', + query: { + bool: { + should: [ + { + match: { + 'threat.enrichments.matched.field': 'file.hash.md5', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - score_mode: 'none', - ignore_unmapped: true, }, + score_mode: 'none', + ignore_unmapped: true, }, - ], - }, + }, + ], }, - { - bool: { - should: [ - { - exists: { - // ✅ Non-nested fields are NOT converted to the `nested` query syntax - // ✅ Non-nested fields do NOT include the `ignore_unmapped` option - field: '@timestamp', - }, + }, + { + bool: { + should: [ + { + exists: { + // ✅ Non-nested fields are NOT converted to the `nested` query syntax + // ✅ Non-nested fields do NOT include the `ignore_unmapped` option + field: '@timestamp', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, - { - exists: { - field: '_id', - }, + }, + { + exists: { + field: '_id', }, - ], - should: [], - must_not: [], - }, + }, + ], + should: [], + must_not: [], + }, + }; + + it('should, by default, build a query where the `nested` fields syntax includes the `"ignore_unmapped":true` option', () => { + const [converted, _] = convertToBuildEsQuery({ + config, + dataView: createStubDataView({ spec: {} }), + queries: queryWithNestedFields, + dataViewSpec: mockDataViewSpec, + filters, }); + + expect(JSON.parse(converted ?? '')).to.eql(expectedConverted); }); it('should, when the default is overridden, build a query where `nested` fields include the `"ignore_unmapped":false` option', () => { @@ -280,6 +282,82 @@ describe('convertToBuildEsQuery', () => { }, }); }); + + describe('When ignoreFilterIfFieldNotInIndex is true', () => { + const updatedConfig = { ...config, ignoreFilterIfFieldNotInIndex: true }; + + it('should use dataViewSpec when an empty dataView is provided', () => { + mockDataViewSpec.fields = { + _id: { + name: '_id', + type: 'string', + esTypes: ['keyword'], + aggregatable: true, + searchable: true, + scripted: false, + }, + }; + const emptyStubDataView = createStubDataView({ spec: { id: '', title: '' } }); + const [converted] = convertToBuildEsQuery({ + config: updatedConfig, + dataView: emptyStubDataView, // <-- empty dataView + queries: queryWithNestedFields, + dataViewSpec: mockDataViewSpec, // <-- should be used instead of the empty dataView + filters, + }); + + expect(JSON.parse(converted ?? '')).to.eql(expectedConverted); // just verify that something was built + }); + + it('should not use the field if the filter is not mapped in the', () => { + const updatedConvertedWithoutIdQuery = structuredClone(expectedConverted); + updatedConvertedWithoutIdQuery.bool.filter = [updatedConvertedWithoutIdQuery.bool.filter[0]]; // remove the search bar filter + const dataViewWithoutIdMapped = createStubDataView({ + spec: { + id: 'test-id', + title: 'some-title', + }, + }); + const [converted] = convertToBuildEsQuery({ + config: updatedConfig, + dataView: dataViewWithoutIdMapped, + queries: queryWithNestedFields, + dataViewSpec: mockDataViewSpec, + filters, + }); + + expect(JSON.parse(converted ?? '')).to.eql(updatedConvertedWithoutIdQuery); // just verify that something was built + }); + + it('should use the filters when the field is mapped in the dataView', () => { + const dataViewWithIdMapped = createStubDataView({ + spec: { + id: 'test-id', + title: 'some-title', + fields: { + _id: { + name: '_id', + type: 'string', + esTypes: ['keyword'], + aggregatable: true, + searchable: true, + scripted: false, + }, + }, + }, + }); + const [converted] = convertToBuildEsQuery({ + config: updatedConfig, + dataView: dataViewWithIdMapped, + queries: queryWithNestedFields, + dataViewSpec: mockDataViewSpec, + filters, + }); + + // This should have the id with the + expect(JSON.parse(converted ?? '')).to.eql(expectedConverted); + }); + }); }); describe('buildGlobalQuery', () => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.ts b/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.ts index ecc90b7f13985..26d6e2db69813 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/common/lib/kuery/index.ts @@ -210,7 +210,7 @@ export const dataViewSpecToViewBase = (dataViewSpec?: DataViewSpec): DataViewBas export const convertToBuildEsQuery = ({ config, - dataView, // New dataview prepended with feature flag to enable easy cleanup + dataView, // New dataview with newDataViewPickerEnabled dataViewSpec, // Account for the case where sourcerer is active, but this can just use dataView queries, filters, @@ -225,10 +225,11 @@ export const convertToBuildEsQuery = ({ filters: Filter[]; }): [string, undefined] | [undefined, Error] => { try { + const newDataViewExists = dataView?.id && dataView?.getIndexPattern(); return [ JSON.stringify( buildEsQuery( - dataView ?? (dataViewSpecToViewBase(dataViewSpec) as DataView), + newDataViewExists ? dataView : (dataViewSpecToViewBase(dataViewSpec) as DataView), queries, filters.filter((f) => f.meta.disabled === false), {