From b4099943224b866904d92c702572a3b6db0c3f06 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Fri, 8 Aug 2025 17:26:05 -0400 Subject: [PATCH 1/2] [ResponseOps][Reporting] reapply index mappings of reporting index at startup resolves: https://github.com/elastic/kibana/issues/231200 This PR adds code run at startup to: - read the `.kibana-reporting` index template to get the mappings - apply those mappings directly to the write index of the `.kibana-reporting` data stream This is needed for those times when the mappings change, for instance in this PR: https://github.com/elastic/elasticsearch/pull/129061. The mappings do get updated, but are not applied to the data stream, so in fact the "new" fields (when updates happen) are not actually mapped in the existing data stream. I was hoping to actually roll over the index, but it's not clear how we could determine if we need to, and we certainly don't want to do that on every restart. Re-applying the mappings will ensure that any changes to the index template's mapping are made to the current index. --- .../private/kbn-reporting/server/constants.ts | 2 + .../reporting/server/lib/store/store.test.ts | 30 +++++++++- .../reporting/server/lib/store/store.ts | 59 ++++++++++++++++++- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/platform/packages/private/kbn-reporting/server/constants.ts b/src/platform/packages/private/kbn-reporting/server/constants.ts index 3b36a04ec8552..a93b81edfaf76 100644 --- a/src/platform/packages/private/kbn-reporting/server/constants.ts +++ b/src/platform/packages/private/kbn-reporting/server/constants.ts @@ -21,6 +21,8 @@ export const REPORTING_DATA_STREAM_WILDCARD = '.kibana-reporting*'; export const REPORTING_LEGACY_INDICES = '.reporting-*'; // Used to search for all reports and check for managing privileges export const REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY = '.reporting-*,.kibana-reporting*'; +// Name of index template +export const REPORTING_DATA_STREAM_INDEX_TEMPLATE = '.kibana-reporting'; // Name of component template which Kibana overrides for lifecycle settings export const REPORTING_DATA_STREAM_COMPONENT_TEMPLATE = 'kibana-reporting@custom'; diff --git a/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts b/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts index 61d0a636e19f2..944875fc9c554 100644 --- a/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts @@ -13,10 +13,12 @@ import { Report, ReportingStore, SavedReport } from '.'; import { ReportingCore } from '../..'; import { createMockReportingCore } from '../../test_helpers'; +type MockEsClient = ReturnType; + describe('ReportingStore', () => { const mockLogger = loggingSystemMock.createLogger(); let mockCore: ReportingCore; - let mockEsClient: ReturnType; + let mockEsClient: MockEsClient; beforeEach(async () => { const reportingConfig = { @@ -455,6 +457,7 @@ describe('ReportingStore', () => { it('creates an ILM policy for managing reporting indices if there is not already one', async () => { mockEsClient.ilm.getLifecycle.mockRejectedValue({ statusCode: 404 }); mockEsClient.ilm.putLifecycle.mockResponse({} as any); + mockCallsForApplyingMapping(mockEsClient); const store = new TestReportingStore(mockCore, mockLogger); const createIlmPolicySpy = jest.spyOn(store, 'createIlmPolicy'); @@ -478,6 +481,7 @@ describe('ReportingStore', () => { it('does not create an ILM policy for managing reporting indices if one already exists', async () => { mockEsClient.ilm.getLifecycle.mockResponse({}); + mockCallsForApplyingMapping(mockEsClient); const store = new TestReportingStore(mockCore, mockLogger); const createIlmPolicySpy = jest.spyOn(store, 'createIlmPolicy'); @@ -493,6 +497,7 @@ describe('ReportingStore', () => { statefulSettings: { enabled: false }, }; mockCore = await createMockReportingCore(createMockConfigSchema(reportingConfig)); + mockCallsForApplyingMapping(mockEsClient); const store = new TestReportingStore(mockCore, mockLogger); const createIlmPolicySpy = jest.spyOn(store, 'createIlmPolicy'); @@ -502,3 +507,26 @@ describe('ReportingStore', () => { }); }); }); + +function mockCallsForApplyingMapping(mockEsClient: MockEsClient) { + mockEsClient.indices.exists.mockResponse(true); + mockEsClient.indices.getIndexTemplate.mockResponse({ + index_templates: [ + { + name: '.kibana-reporting', + index_template: { + template: { + mappings: { + properties: { + foo: { + type: 'keyword', + }, + }, + }, + }, + }, + }, + ], + } as any); + mockEsClient.indices.putMapping.mockResponse(undefined as any); +} diff --git a/x-pack/platform/plugins/private/reporting/server/lib/store/store.ts b/x-pack/platform/plugins/private/reporting/server/lib/store/store.ts index 38fd5a6036c7c..4c845087a856e 100644 --- a/x-pack/platform/plugins/private/reporting/server/lib/store/store.ts +++ b/x-pack/platform/plugins/private/reporting/server/lib/store/store.ts @@ -17,6 +17,7 @@ import type { import { REPORTING_DATA_STREAM_ALIAS, REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, + REPORTING_DATA_STREAM_INDEX_TEMPLATE, } from '@kbn/reporting-server'; import moment from 'moment'; import type { Report } from '.'; @@ -170,12 +171,66 @@ export class ReportingStore { await this.createIlmPolicy(); } } catch (e) { - this.logger.error('Error in start phase'); - this.logger.error(e); + this.logger.error(`Error creating ILM policy: ${e.message}`, { + error: { stack_trace: e.stack }, + }); + throw e; + } + + try { + await this.reapplyMappings(); + } catch (e) { + this.logger.error(`Error applying mappings: ${e.message}`, { + error: { stack_trace: e.stack }, + }); throw e; } } + private async reapplyMappings(): Promise { + const client = await this.getClient(); + + const exists = await client.indices.exists({ + index: REPORTING_DATA_STREAM_ALIAS, + }); + + if (!exists) { + this.logger.info( + `Mappings not applied to ${REPORTING_DATA_STREAM_ALIAS} since it does not exist.` + ); + return; + } + + const gotTemplate = await client.indices.getIndexTemplate({ + name: REPORTING_DATA_STREAM_INDEX_TEMPLATE, + }); + if (gotTemplate.index_templates.length === 0) { + throw new Error(`No index template found for ${REPORTING_DATA_STREAM_INDEX_TEMPLATE}`); + } + + const template = gotTemplate.index_templates.find( + (t) => t.name === REPORTING_DATA_STREAM_INDEX_TEMPLATE + ); + if (!template) { + throw new Error( + `No matching index template found for ${REPORTING_DATA_STREAM_INDEX_TEMPLATE}` + ); + } + + const mappings = template.index_template.template?.mappings?.properties; + if (!mappings) { + throw new Error(`No mappings found for ${REPORTING_DATA_STREAM_INDEX_TEMPLATE}`); + } + + await client.indices.putMapping({ + index: REPORTING_DATA_STREAM_ALIAS, + properties: mappings, + write_index_only: true, + }); + + this.logger.info(`Mappings applied to ${REPORTING_DATA_STREAM_ALIAS}`); + } + public async addReport(report: Report): Promise { try { report.updateWithEsDoc(await this.indexReport(report)); From 4d79b159fd70ebf30ec1af9eb36feec0829166e3 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Mon, 11 Aug 2025 13:22:01 -0400 Subject: [PATCH 2/2] fix jest test --- .../plugins/private/reporting/server/lib/store/store.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts b/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts index 944875fc9c554..0fe37c27ed23a 100644 --- a/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/lib/store/store.test.ts @@ -497,6 +497,8 @@ describe('ReportingStore', () => { statefulSettings: { enabled: false }, }; mockCore = await createMockReportingCore(createMockConfigSchema(reportingConfig)); + mockEsClient = (await mockCore.getEsClient()).asInternalUser as typeof mockEsClient; + mockCallsForApplyingMapping(mockEsClient); const store = new TestReportingStore(mockCore, mockLogger);