diff --git a/src/platform/packages/shared/kbn-esql-utils/index.ts b/src/platform/packages/shared/kbn-esql-utils/index.ts index 669c145111175..dbac398f0a1c1 100644 --- a/src/platform/packages/shared/kbn-esql-utils/index.ts +++ b/src/platform/packages/shared/kbn-esql-utils/index.ts @@ -69,6 +69,7 @@ export { isComputedColumn, getQuerySummary, getEsqlControls, + injectMetadataFields, type ESQLStatsQueryMeta, } from './src'; diff --git a/src/platform/packages/shared/kbn-esql-utils/src/index.ts b/src/platform/packages/shared/kbn-esql-utils/src/index.ts index 1b8d6d846881e..f315d19d0c0cf 100644 --- a/src/platform/packages/shared/kbn-esql-utils/src/index.ts +++ b/src/platform/packages/shared/kbn-esql-utils/src/index.ts @@ -36,6 +36,7 @@ export { getSourceCommandFromESQLQuery, } from './utils/get_index_pattern_from_query'; export { queryCannotBeSampled } from './utils/query_cannot_be_sampled'; +export { injectMetadataFields } from './utils/inject_metadata_fields'; export { appendToESQLQuery } from './utils/append_to_query/utils'; export { appendStatsByToQuery } from './utils/append_to_query/append_stats_by'; export { appendWhereClauseToESQLQuery } from './utils/append_to_query/append_where'; diff --git a/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.test.ts b/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.test.ts new file mode 100644 index 0000000000000..d5b52a8363396 --- /dev/null +++ b/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.test.ts @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { injectMetadataFields } from './inject_metadata_fields'; + +describe('injectMetadataFields', () => { + describe('single field (_id) — parity with Security injectMetadataId', () => { + describe('METADATA injection into FROM', () => { + it('injects METADATA when FROM has no metadata', () => { + expect(injectMetadataFields('FROM logs*', ['_id'])).toBe('FROM logs* METADATA _id'); + }); + + it('preserves query when field already exists', () => { + expect(injectMetadataFields('FROM logs* METADATA _id', ['_id'])).toBe( + 'FROM logs* METADATA _id' + ); + }); + + it('preserves query when field exists with other fields', () => { + expect(injectMetadataFields('FROM logs* METADATA _id, _version, _index', ['_id'])).toBe( + 'FROM logs* METADATA _id, _version, _index' + ); + }); + + it('appends field when METADATA exists without it', () => { + expect(injectMetadataFields('FROM logs* METADATA _index', ['_id'])).toBe( + 'FROM logs* METADATA _index, _id' + ); + }); + + it('injects METADATA with multiple source indices', () => { + expect(injectMetadataFields('FROM logs*, other-index*', ['_id'])).toBe( + 'FROM logs*, other-index* METADATA _id' + ); + }); + + it('injects METADATA before pipe commands', () => { + expect(injectMetadataFields('FROM logs* | WHERE x > 5', ['_id'])).toBe( + 'FROM logs* METADATA _id | WHERE x > 5' + ); + }); + + it('handles trailing whitespace', () => { + expect(injectMetadataFields('FROM logs* ', ['_id'])).toBe('FROM logs* METADATA _id'); + }); + + it('handles multi-line query', () => { + expect(injectMetadataFields('FROM packetbeat*\n | LIMIT 100', ['_id'])).toBe( + 'FROM packetbeat* METADATA _id | LIMIT 100' + ); + }); + }); + + describe('KEEP best-effort fix', () => { + it('adds field to KEEP when missing', () => { + expect(injectMetadataFields('FROM logs* METADATA _id | KEEP agent.name', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP agent.name, _id' + ); + }); + + it('does not add field to KEEP when already present', () => { + expect( + injectMetadataFields('FROM logs* METADATA _id | KEEP agent.name, _id', ['_id']) + ).toBe('FROM logs* METADATA _id | KEEP agent.name, _id'); + }); + + it('adds field to KEEP with partial wildcard', () => { + expect(injectMetadataFields('FROM logs* | KEEP agent.*', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP agent.*, _id' + ); + }); + + it('handles KEEP followed by other commands', () => { + expect(injectMetadataFields('FROM logs* | KEEP a, b | EVAL c = a', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP a, b, _id | EVAL c = a' + ); + }); + + it('handles KEEP with field already present and no METADATA', () => { + expect(injectMetadataFields('FROM logs* | KEEP agent.name, _id', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP agent.name, _id' + ); + }); + }); + + describe('DROP (accepted limitation)', () => { + it('does not modify DROP — accepted limitation', () => { + expect(injectMetadataFields('FROM logs* | DROP _id', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP _id' + ); + }); + + it('injects METADATA but does not remove explicit DROP', () => { + expect(injectMetadataFields('FROM logs* METADATA _id | DROP _id', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP _id' + ); + }); + + it('does not inject field into KEEP that appears after DROP', () => { + expect(injectMetadataFields('FROM logs* | DROP _id | KEEP agent.name', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP _id | KEEP agent.name' + ); + }); + + it('injects field into KEEP that appears before DROP', () => { + expect(injectMetadataFields('FROM logs* | KEEP agent.name | DROP _id', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP agent.name, _id | DROP _id' + ); + }); + + it('DROP multiple fields including target stops KEEP injection downstream', () => { + expect(injectMetadataFields('FROM logs* | DROP _id, agent.type | KEEP host', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP _id, agent.type | KEEP host' + ); + }); + + it('DROP without target does not affect KEEP injection', () => { + expect( + injectMetadataFields('FROM logs* | DROP agent.type | KEEP agent.name', ['_id']) + ).toBe('FROM logs* METADATA _id | DROP agent.type | KEEP agent.name, _id'); + }); + + it('DROP with wildcard _* stops KEEP injection', () => { + expect(injectMetadataFields('FROM logs* | DROP _* | KEEP agent.name', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP _* | KEEP agent.name' + ); + }); + + it('DROP with global wildcard * stops KEEP injection', () => { + expect(injectMetadataFields('FROM logs* | DROP * | KEEP agent.name', ['_id'])).toBe( + 'FROM logs* METADATA _id | DROP * | KEEP agent.name' + ); + }); + }); + + describe('RENAME (stops KEEP injection)', () => { + it('does not inject field into KEEP after RENAME', () => { + expect( + injectMetadataFields('FROM logs* | RENAME _id AS doc_id | KEEP agent.name', ['_id']) + ).toBe('FROM logs* METADATA _id | RENAME _id AS doc_id | KEEP agent.name'); + }); + + it('injects field into KEEP before RENAME', () => { + expect( + injectMetadataFields('FROM logs* | KEEP agent.name | RENAME _id AS doc_id', ['_id']) + ).toBe('FROM logs* METADATA _id | KEEP agent.name, _id | RENAME _id AS doc_id'); + }); + + it('RENAME of a different column does not affect KEEP injection', () => { + expect( + injectMetadataFields('FROM logs* | RENAME host AS hostname | KEEP hostname', ['_id']) + ).toBe('FROM logs* METADATA _id | RENAME host AS hostname | KEEP hostname, _id'); + }); + + it('RENAME mid-pipeline stops injection for downstream KEEP', () => { + expect( + injectMetadataFields('FROM logs* | KEEP a, _id | RENAME _id AS my_id | KEEP a, my_id', [ + '_id', + ]) + ).toBe('FROM logs* METADATA _id | KEEP a, _id | RENAME _id AS my_id | KEEP a, my_id'); + }); + + it('RENAME other_col AS target does NOT stop KEEP injection', () => { + expect( + injectMetadataFields('FROM logs* | RENAME doc_id AS _id | KEEP agent.name', ['_id']) + ).toBe('FROM logs* METADATA _id | RENAME doc_id AS _id | KEEP agent.name, _id'); + }); + }); + + describe('EVAL (stops KEEP injection)', () => { + it('does not inject field into KEEP after EVAL assignment', () => { + expect( + injectMetadataFields('FROM logs* | EVAL _id = "overwritten" | KEEP agent.name', ['_id']) + ).toBe('FROM logs* METADATA _id | EVAL _id = "overwritten" | KEEP agent.name'); + }); + + it('injects field into KEEP before EVAL assignment', () => { + expect( + injectMetadataFields('FROM logs* | KEEP agent.name | EVAL _id = "overwritten"', ['_id']) + ).toBe('FROM logs* METADATA _id | KEEP agent.name, _id | EVAL _id = "overwritten"'); + }); + + it('EVAL of a different column does not affect KEEP injection', () => { + expect(injectMetadataFields('FROM logs* | EVAL x = 1 | KEEP x', ['_id'])).toBe( + 'FROM logs* METADATA _id | EVAL x = 1 | KEEP x, _id' + ); + }); + + it('EVAL mid-pipeline stops injection for downstream KEEP', () => { + expect( + injectMetadataFields('FROM logs* | KEEP a, _id | EVAL _id = "test" | KEEP a, _id', [ + '_id', + ]) + ).toBe('FROM logs* METADATA _id | KEEP a, _id | EVAL _id = "test" | KEEP a, _id'); + }); + }); + + describe('DISSECT/GROK (does not stop KEEP injection)', () => { + it('injects field into KEEP after DISSECT', () => { + expect( + injectMetadataFields('FROM logs* | DISSECT message "%{parsed}" | KEEP parsed', ['_id']) + ).toBe('FROM logs* METADATA _id | DISSECT message "%{parsed}" | KEEP parsed, _id'); + }); + + it('injects field into KEEP after GROK', () => { + expect( + injectMetadataFields('FROM logs* | GROK message "%{WORD:parsed}" | KEEP parsed', ['_id']) + ).toBe('FROM logs* METADATA _id | GROK message "%{WORD:parsed}" | KEEP parsed, _id'); + }); + + it('injects field into KEEP before DISSECT', () => { + expect( + injectMetadataFields('FROM logs* | KEEP agent.name | DISSECT message "%{parsed}"', [ + '_id', + ]) + ).toBe('FROM logs* METADATA _id | KEEP agent.name, _id | DISSECT message "%{parsed}"'); + }); + }); + + describe('KEEP with wildcards', () => { + it('adds field to KEEP * (redundant but harmless)', () => { + expect(injectMetadataFields('FROM logs* | KEEP *', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP *, _id' + ); + }); + + it('adds field to KEEP _* (redundant but harmless)', () => { + expect(injectMetadataFields('FROM logs* | KEEP _*', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP _*, _id' + ); + }); + }); + + describe('multiple KEEP commands', () => { + it('injects field into both KEEP commands', () => { + expect(injectMetadataFields('FROM logs* | KEEP a, b | KEEP a', ['_id'])).toBe( + 'FROM logs* METADATA _id | KEEP a, b, _id | KEEP a, _id' + ); + }); + }); + + describe('lowercase commands', () => { + it('normalizes lowercase commands to uppercase in output', () => { + expect( + injectMetadataFields( + 'from logs* metadata _index | where x > 5 | keep agent.name | limit 10', + ['_id'] + ) + ).toBe('FROM logs* METADATA _index, _id | WHERE x > 5 | KEEP agent.name, _id | LIMIT 10'); + }); + }); + }); + + describe('multiple fields (_id + _index) — Discover use case', () => { + it('adds both fields when no METADATA clause exists', () => { + expect(injectMetadataFields('FROM logs-*', ['_id', '_index'])).toBe( + 'FROM logs-* METADATA _id, _index' + ); + }); + + it('adds only missing fields when some already present', () => { + expect(injectMetadataFields('FROM logs-* METADATA _id | LIMIT 20', ['_id', '_index'])).toBe( + 'FROM logs-* METADATA _id, _index | LIMIT 20' + ); + }); + + it('does not duplicate when all requested fields already present', () => { + const query = 'FROM logs-* METADATA _id, _index | LIMIT 20'; + expect(injectMetadataFields(query, ['_id', '_index'])).toBe(query); + }); + + it('preserves existing extra metadata fields', () => { + expect( + injectMetadataFields('FROM logs-* METADATA _ignored | LIMIT 20', ['_id', '_index']) + ).toBe('FROM logs-* METADATA _ignored, _id, _index | LIMIT 20'); + }); + + it('adds both fields to KEEP when missing', () => { + expect( + injectMetadataFields('FROM logs-* | KEEP @timestamp, message', ['_id', '_index']) + ).toBe('FROM logs-* METADATA _id, _index | KEEP @timestamp, message, _id, _index'); + }); + + it('handles independent DROP/RENAME/EVAL per field', () => { + const result = injectMetadataFields('FROM logs-* | DROP _id | KEEP @timestamp', [ + '_id', + '_index', + ]); + expect(result).toContain('METADATA _id, _index'); + expect(result).toContain('KEEP @timestamp, _index'); + expect(result).not.toMatch(/KEEP.*_id/); + }); + + it('does not modify queries without FROM', () => { + expect(injectMetadataFields('ROW a = 1', ['_id', '_index'])).not.toContain('METADATA'); + }); + + it('handles malformed input without crashing', () => { + expect(() => injectMetadataFields('NOT VALID ESQL {{{}}}', ['_id'])).not.toThrow(); + }); + }); +}); diff --git a/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.ts b/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.ts new file mode 100644 index 0000000000000..fcc7c2c23db31 --- /dev/null +++ b/src/platform/packages/shared/kbn-esql-utils/src/utils/inject_metadata_fields.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { ESQLAstQueryExpression, ESQLAstItem, ESQLFunction } from '@elastic/esql/types'; +import { + BasicPrettyPrinter, + Builder, + Parser, + mutate, + isColumn, + isFunctionExpression, +} from '@elastic/esql'; + +/** + * Injects the specified metadata fields into an ES|QL query's FROM command + * and performs best-effort fixups so they survive through the pipeline. + * + * 1. Upserts each field into `FROM ... METADATA ...` (idempotent). + * 2. Appends each field to any KEEP command that would otherwise exclude it. + * + * KEEP injection stops conservatively at the first command where the field + * may no longer hold its original metadata value: + * - `DROP ` / wildcard DROP + * - `RENAME AS …` + * - `EVAL = …` + * + * Falls back to the original query string on parse errors. + */ +export function injectMetadataFields(esqlQuery: string, fields: string[]): string { + const { root } = Parser.parse(esqlQuery); + for (const field of fields) { + mutate.commands.from.metadata.upsert(root, field); + addFieldToKeepCommands(root, field); + } + return BasicPrettyPrinter.print(root); +} + +/** + * Walks the pipeline and appends `fieldName` to KEEP commands that don't + * already include it. Stops at the first command where the field may no + * longer hold its original metadata value (DROP, RENAME, or EVAL of it). + */ +function addFieldToKeepCommands(root: ESQLAstQueryExpression, fieldName: string): void { + if (!root.commands.some((cmd) => cmd.name === 'keep')) { + return; + } + + for (const cmd of root.commands) { + if (cmd.name === 'drop' && hasColumnMatching(cmd.args, fieldName)) { + break; + } + if (cmd.name === 'rename' && hasRenameOf(cmd.args, fieldName)) { + break; + } + if (cmd.name === 'eval' && hasAssignmentTo(cmd.args, fieldName)) { + break; + } + if (cmd.name === 'keep') { + if (!cmd.args.some((arg) => isColumn(arg) && arg.name === fieldName)) { + cmd.args.push(Builder.expression.column(fieldName)); + } + } + } +} + +function isTargetingColumn(arg: ESQLAstItem, columnName: string): arg is ESQLFunction { + return isFunctionExpression(arg) && isColumn(arg.args[0]) && arg.args[0].name === columnName; +} + +function hasRenameOf(args: ESQLAstItem[], fieldName: string): boolean { + return args.some( + (arg) => isTargetingColumn(arg, fieldName) && (arg.name === 'as' || arg.name === '=') + ); +} + +function hasAssignmentTo(args: ESQLAstItem[], fieldName: string): boolean { + return args.some((arg) => isTargetingColumn(arg, fieldName) && arg.name === '='); +} + +function hasColumnMatching(args: readonly ESQLAstItem[], fieldName: string): boolean { + return args.some((arg) => isColumn(arg) && (arg.name === fieldName || arg.name.includes('*'))); +} diff --git a/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.test.ts b/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.test.ts index 02434fe0a13cd..4428d0cc47a39 100644 --- a/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.test.ts +++ b/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.test.ts @@ -33,32 +33,30 @@ describe('fetchEsql', () => { scopedProfilesManager, }; + const mockExpressionResult = (columns: unknown[], rows: unknown[]) => { + jest.spyOn(discoverServiceMock.expressions, 'execute').mockReturnValueOnce({ + cancel: jest.fn(), + getData: jest.fn(() => of({ result: { columns, rows } })), + } as unknown as ExecutionContract); + }; + + const col = (id: string, type = 'keyword') => ({ id, name: id, meta: { type } }); + it('resolves with returned records', async () => { const hits = [ - { _id: '1', foo: 'bar' }, - { _id: '2', foo: 'baz' }, + { name: 'one', foo: 'bar' }, + { name: 'two', foo: 'baz' }, ] as unknown as EsHitRecord[]; const records = hits.map((hit, i) => ({ id: String(i), raw: hit, flattened: hit, })); - const expressionsExecuteSpy = jest.spyOn(discoverServiceMock.expressions, 'execute'); - expressionsExecuteSpy.mockReturnValueOnce({ - cancel: jest.fn(), - getData: jest.fn(() => - of({ - result: { - columns: ['_id', 'foo'], - rows: hits, - }, - }) - ), - } as unknown as ExecutionContract); + mockExpressionResult(['name', 'foo'], hits); const resolveDocumentProfileSpy = jest.spyOn(scopedProfilesManager, 'resolveDocumentProfile'); expect(await fetchEsql(fetchEsqlMockProps)).toEqual({ records, - esqlQueryColumns: ['_id', 'foo'], + esqlQueryColumns: ['name', 'foo'], esqlHeaderWarning: undefined, interceptedWarnings: [], }); @@ -67,6 +65,30 @@ describe('fetchEsql', () => { expect(resolveDocumentProfileSpy).toHaveBeenCalledWith({ record: records[1] }); }); + it('should hide injected metadata from enumerable properties but keep accessible', async () => { + const hits = [ + { _id: 'doc1', _index: 'logs-1', message: 'hello' }, + { _id: 'doc2', _index: 'logs-1', message: 'world' }, + ] as unknown as EsHitRecord[]; + mockExpressionResult([col('_id'), col('_index'), col('message')], hits); + const result = await fetchEsql(fetchEsqlMockProps); + expect(result.esqlQueryColumns).toEqual([col('message')]); + expect(Object.keys(result.records[0].raw)).toEqual(['message']); + expect(result.records[0].raw._id).toBe('doc1'); + expect(result.records[0].raw._index).toBe('logs-1'); + }); + + it('should not filter metadata columns that the user explicitly requested', async () => { + const hits = [{ _id: 'doc1', message: 'hello' }] as unknown as EsHitRecord[]; + mockExpressionResult([col('_id'), col('message')], hits); + const result = await fetchEsql({ + ...fetchEsqlMockProps, + query: { esql: 'from * metadata _id' }, + }); + expect(result.esqlQueryColumns).toEqual([col('_id'), col('message')]); + expect(Object.keys(result.records[0].raw)).toContain('_id'); + }); + it('should use inputTimeRange if provided', () => { const timeRange: TimeRange = { from: 'now-15m', to: 'now' }; const result = getTextBasedQueryStateToAstProps({ ...fetchEsqlMockProps, timeRange }); diff --git a/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.ts b/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.ts index e9f8e64c0b1c0..465071b55bf6c 100644 --- a/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.ts +++ b/src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_esql.ts @@ -11,6 +11,7 @@ import { pluck } from 'rxjs'; import { lastValueFrom } from 'rxjs'; import { i18n } from '@kbn/i18n'; import type { Query, AggregateQuery, Filter, TimeRange, ProjectRouting } from '@kbn/es-query'; +import { isOfAggregateQueryType } from '@kbn/es-query'; import type { Adapters } from '@kbn/inspector-plugin/common'; import type { ESQLControlVariable } from '@kbn/esql-types'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; @@ -20,6 +21,7 @@ import type { DataView } from '@kbn/data-views-plugin/common'; import { textBasedQueryStateToAstWithValidation } from '@kbn/data-plugin/common'; import type { DataTableRecord } from '@kbn/discover-utils'; import type { SearchResponseWarning } from '@kbn/search-response-warnings'; +import { injectMetadataFields, retrieveMetadataColumns } from '@kbn/esql-utils'; import type { RecordsFetchResponse } from '../../types'; import type { ScopedProfilesManager } from '../../../context_awareness'; @@ -66,8 +68,27 @@ export function fetchEsql({ projectRouting, inspectorConfig, }: FetchEsqlParams): Promise { + const metadataFieldsToInject = ['_id', '_index']; + let queryWithMetadata = query; + let injectedMetadataFields: string[] = []; + if (isOfAggregateQueryType(query)) { + try { + const existingMetadata = retrieveMetadataColumns(query.esql); + injectedMetadataFields = metadataFieldsToInject.filter( + (field) => !existingMetadata.includes(field) + ); + if (injectedMetadataFields.length > 0) { + const injected = injectMetadataFields(query.esql, metadataFieldsToInject); + queryWithMetadata = { ...query, esql: injected }; + } + } catch (error) { + // eslint-disable-next-line no-console + console.warn('[Discover] Failed to inject METADATA _id, _index into ES|QL query:', error); + } + } + const props = getTextBasedQueryStateToAstProps({ - query, + query: queryWithMetadata, inputQuery, filters, timeRange, @@ -102,13 +123,29 @@ export function fetchEsql({ } else { const table = response as Datatable; const rows = table?.rows ?? []; - esqlQueryColumns = table?.columns ?? undefined; + esqlQueryColumns = table?.columns?.filter( + (col) => !injectedMetadataFields.includes(col.id) + ); esqlHeaderWarning = table.warning ?? undefined; + // Make injected metadata fields non-enumerable so Object.keys() skips them + // in grid/sidebar column detection, while still accessible via direct property + // access for features like Log AI Insight. finalData = rows.map((row, idx) => { + let raw = row; + if (injectedMetadataFields.length > 0) { + raw = { ...row }; + for (const field of injectedMetadataFields) { + if (field in raw) { + const value = raw[field]; + delete raw[field]; + Object.defineProperty(raw, field, { value, enumerable: false }); + } + } + } const record: DataTableRecord = { id: String(idx), - raw: row, - flattened: row, + raw, + flattened: raw, }; return scopedProfilesManager.resolveDocumentProfile({ record }); diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/public/components/insights/index.tsx b/x-pack/solutions/observability/plugins/observability_agent_builder/public/components/insights/index.tsx index 3b89948241b41..35e04324096d5 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/public/components/insights/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/public/components/insights/index.tsx @@ -83,15 +83,20 @@ export function createLogAIInsight( export const createLogsAIInsightRenderer = (LogAIInsightRender: ReturnType) => ({ doc }: ObservabilityLogsAiInsightFeatureRenderDeps) => { - const mappedDoc = useMemo( - () => ({ - fields: Object.entries(doc.flattened).map(([field, value]) => ({ - field, - value: Array.isArray(value) ? value : [value], - })), - }), - [doc] - ); + const mappedDoc = useMemo(() => { + const fields = Object.entries(doc.flattened).map(([field, value]) => ({ + field, + value: Array.isArray(value) ? value : [value], + })); + const raw = doc.raw as Record; + if (raw._id !== undefined) { + fields.push({ field: '_id', value: [raw._id] }); + } + if (raw._index !== undefined) { + fields.push({ field: '_index', value: [raw._index] }); + } + return { fields }; + }, [doc]); return ; };