From 3e445697c741a74dafeb9a5ffbea6e7dd1b2f7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 8 Dec 2025 16:30:24 +0100 Subject: [PATCH 1/2] Add datastream lifecycle support to indices metadata --- .../indices_metadata/server/lib/ebt/events.ts | 13 +++++++++++++ .../server/lib/services/indices_metadata.ts | 1 + .../server/lib/services/indices_metadata.types.ts | 7 +++++++ .../server/lib/services/receiver.test.ts | 15 ++++++++++++++- .../server/lib/services/receiver.ts | 12 +++++++++++- 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts index f09863c13ff69..f8322f035f557 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts @@ -31,6 +31,19 @@ export const DATA_STREAM_EVENT: EventTypeOpts = { type: 'keyword', _meta: { optional: true, description: 'ILM policy associated to the datastream' }, }, + dsl: { + properties: { + enabled: { + type: 'boolean', + _meta: { description: 'Whether the data stream is enabled' }, + }, + data_retention: { + type: 'text', + _meta: { optional: true, description: 'Data retention period' }, + }, + }, + _meta: { optional: true, description: 'Data stream lifecycle settings' }, + }, template: { type: 'keyword', _meta: { optional: true, description: 'Template associated to the datastream' }, diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts index caf5a13cbdd78..7950bd3f966f4 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts @@ -257,6 +257,7 @@ export class IndicesMetadataService { this.logger.debug('Indices settings sent', { count: indicesSettings.items.length } as LogMeta); return indicesSettings.items.length; } + private async publishIlmStats(indices: string[]): Promise> { const ilmNames = new Set(); const ilmsStats: IlmsStats = { diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts index 9d5264cbd89e7..73d559fcf4dbd 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts @@ -114,9 +114,16 @@ export interface Index { export interface DataStreams { items: DataStream[]; } + +export interface DataStreamLifeCycle { + enabled: boolean; + data_retention?: string; +} + export interface DataStream { datastream_name: string; ilm_policy?: string; + dsl?: DataStreamLifeCycle; template?: string; indices?: Index[]; } diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts index b7846d1e1d761..c5cb3794d5c46 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts @@ -158,12 +158,21 @@ describe('Indices Metadata - MetadataReceiver', () => { expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name: '*', expand_wildcards: ['open', 'hidden'], - filter_path: ['data_streams.name', 'data_streams.indices'], + filter_path: [ + 'data_streams.name', + 'data_streams.indices', + 'data_streams.lifecycle.enabled', + 'data_streams.lifecycle.data_retention', + ], }); expect(result).toEqual([ { datastream_name: 'test-datastream', + dsl: { + enabled: false, + data_retention: undefined, + }, indices: [ { index_name: 'test-index-1', @@ -203,6 +212,10 @@ describe('Indices Metadata - MetadataReceiver', () => { expect(result).toEqual([ { datastream_name: 'test-datastream', + dsl: { + enabled: false, + data_retention: undefined, + }, indices: [], }, ]); diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts index 69f784f164b39..aadfcba55449a 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts @@ -89,7 +89,12 @@ export class MetadataReceiver { const request: IndicesGetDataStreamRequest = { name: '*', expand_wildcards: ['open', 'hidden'], - filter_path: ['data_streams.name', 'data_streams.indices'], + filter_path: [ + 'data_streams.name', + 'data_streams.indices', + 'data_streams.lifecycle.enabled', + 'data_streams.lifecycle.data_retention', + ], }; return this.esClient.indices @@ -97,8 +102,13 @@ export class MetadataReceiver { .then((response) => { const streams = response.data_streams ?? []; return streams.map((ds) => { + const dsl = ds.lifecycle; return { datastream_name: ds.name, + dsl: { + enabled: dsl?.enabled ?? false, + data_retention: dsl?.data_retention, + }, indices: ds.indices?.map((index) => { return { From f87aaf03fc6598c187285fa734af4e93c85b01a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 8 Dec 2025 16:56:24 +0100 Subject: [PATCH 2/2] Add tests --- .../lib/services/indices_metadata.test.ts | 168 +++++++++++ .../server/lib/services/receiver.test.ts | 269 ++++++++++++++++++ 2 files changed, 437 insertions(+) diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts index 3bdf69261fc18..32ddecb1ff6d2 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts @@ -71,6 +71,10 @@ describe('Indices Metadata - IndicesMetadataService', () => { const mockDataStreams: DataStream[] = [ { datastream_name: 'test-datastream', + dsl: { + enabled: true, + data_retention: '30d', + }, indices: [{ index_name: 'test-index-1', ilm_policy: 'policy1' }], }, ]; @@ -365,6 +369,60 @@ describe('Indices Metadata - IndicesMetadataService', () => { expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1']); }); + it('should publish datastreams with DSL through full metadata flow', async () => { + const datastreamsWithDsl: DataStream[] = [ + { + datastream_name: 'logs-test', + dsl: { + enabled: true, + data_retention: '90d', + }, + indices: [ + { + index_name: '.ds-logs-test-000001', + ilm_policy: 'logs-policy', + }, + ], + }, + ]; + + receiver.getIndices.mockResolvedValue(mockIndexSettings); + receiver.getDataStreams.mockResolvedValue(datastreamsWithDsl); + receiver.getIndexTemplatesStats.mockResolvedValue(mockIndexTemplates); + receiver.getIndicesStats.mockImplementation(async function* () { + yield* mockIndexStats; + }); + receiver.isIlmStatsAvailable.mockResolvedValue(true); + receiver.getIlmsStats.mockImplementation(async function* () { + yield { + index_name: '.ds-logs-test-000001', + phase: 'hot', + age: '1d', + policy_name: 'logs-policy', + }; + }); + receiver.getIlmsPolicies.mockImplementation(async function* () { + yield { policy_name: 'logs-policy', modified_date: '2023-01-01', phases: {} }; + }); + + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }), + { + items: expect.arrayContaining([ + expect.objectContaining({ + datastream_name: 'logs-test', + dsl: { + enabled: true, + data_retention: '90d', + }, + }), + ]), + } + ); + }); + it('should throw error when not initialized', async () => { const uninitializedService = new IndicesMetadataService(logger, configurationService); @@ -441,6 +499,116 @@ describe('Indices Metadata - IndicesMetadataService', () => { expect(result).toBe(1); expect(logger.debug).toHaveBeenCalledWith('Data streams events sent', { count: 1 }); }); + + it('should publish datastreams with DSL enabled and retention', () => { + const datastreamsWithDsl: DataStream[] = [ + { + datastream_name: 'logs-app-prod', + dsl: { + enabled: true, + data_retention: '7d', + }, + indices: [{ index_name: '.ds-logs-app-prod-000001' }], + }, + { + datastream_name: 'metrics-system', + dsl: { + enabled: false, + data_retention: undefined, + }, + indices: [{ index_name: '.ds-metrics-system-000001' }], + }, + ]; + + const result = service['publishDatastreamsStats'](datastreamsWithDsl); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }), + { + items: [ + expect.objectContaining({ + datastream_name: 'logs-app-prod', + dsl: { + enabled: true, + data_retention: '7d', + }, + }), + expect.objectContaining({ + datastream_name: 'metrics-system', + dsl: { + enabled: false, + data_retention: undefined, + }, + }), + ], + } + ); + expect(result).toBe(2); + }); + + it('should handle datastreams without DSL field', () => { + const datastreamsWithoutDsl: DataStream[] = [ + { + datastream_name: 'legacy-datastream', + indices: [{ index_name: '.ds-legacy-000001' }], + }, + ]; + + const result = service['publishDatastreamsStats'](datastreamsWithoutDsl); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }), + { + items: [ + { + datastream_name: 'legacy-datastream', + indices: [{ index_name: '.ds-legacy-000001' }], + }, + ], + } + ); + expect(result).toBe(1); + }); + + it('should publish mixed DSL configurations', () => { + const mixedDatastreams: DataStream[] = [ + { + datastream_name: 'with-retention', + dsl: { enabled: true, data_retention: '365d' }, + indices: [], + }, + { + datastream_name: 'enabled-no-retention', + dsl: { enabled: true, data_retention: undefined }, + indices: [], + }, + { + datastream_name: 'disabled', + dsl: { enabled: false, data_retention: undefined }, + indices: [], + }, + ]; + + const result = service['publishDatastreamsStats'](mixedDatastreams); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }), + { + items: expect.arrayContaining([ + expect.objectContaining({ + dsl: { enabled: true, data_retention: '365d' }, + }), + expect.objectContaining({ + dsl: { enabled: true, data_retention: undefined }, + }), + expect.objectContaining({ + dsl: { enabled: false, data_retention: undefined }, + }), + ]), + } + ); + expect(result).toBe(3); + }); }); describe('publishIndicesSettings', () => { diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts index c5cb3794d5c46..ebdd5d0bbe76d 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts @@ -228,6 +228,275 @@ describe('Indices Metadata - MetadataReceiver', () => { await expect(receiver.getDataStreams()).rejects.toThrow('Elasticsearch error'); expect(logger.error).toHaveBeenCalledWith('Error fetching datastreams', { error }); }); + + it.each([ + { description: '7 days', retention: '7d' }, + { description: '30 days', retention: '30d' }, + { description: '90 days', retention: '90d' }, + { description: '365 days', retention: '365d' }, + { description: '1 year', retention: '1y' }, + ])('should handle DSL enabled with retention: $description', async ({ retention }) => { + const mockResponse = { + data_streams: [ + { + name: 'logs-test', + lifecycle: { + enabled: true, + data_retention: retention, + }, + indices: [ + { + index_name: '.ds-logs-test-000001', + }, + ], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toEqual([ + { + datastream_name: 'logs-test', + dsl: { + enabled: true, + data_retention: retention, + }, + indices: [ + { + index_name: '.ds-logs-test-000001', + ilm_policy: undefined, + }, + ], + }, + ]); + }); + + it('should handle DSL enabled without retention period', async () => { + const mockResponse = { + data_streams: [ + { + name: 'logs-test', + lifecycle: { + enabled: true, + }, + indices: [ + { + index_name: '.ds-logs-test-000001', + }, + ], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toEqual([ + { + datastream_name: 'logs-test', + dsl: { + enabled: true, + data_retention: undefined, + }, + indices: [ + { + index_name: '.ds-logs-test-000001', + ilm_policy: undefined, + }, + ], + }, + ]); + }); + + it('should handle multiple datastreams with different DSL configurations', async () => { + const mockResponse = { + data_streams: [ + { + name: 'logs-enabled-with-retention', + lifecycle: { + enabled: true, + data_retention: '30d', + }, + indices: [{ index_name: '.ds-logs-1-000001' }], + }, + { + name: 'logs-enabled-no-retention', + lifecycle: { + enabled: true, + }, + indices: [{ index_name: '.ds-logs-2-000001' }], + }, + { + name: 'logs-disabled-no-lifecycle', + indices: [{ index_name: '.ds-logs-3-000001' }], + }, + { + name: 'logs-disabled-with-retention', + lifecycle: { + enabled: false, + data_retention: '7d', + }, + indices: [{ index_name: '.ds-logs-4-000001' }], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toHaveLength(4); + expect(result).toEqual([ + { + datastream_name: 'logs-enabled-with-retention', + dsl: { enabled: true, data_retention: '30d' }, + indices: [{ index_name: '.ds-logs-1-000001', ilm_policy: undefined }], + }, + { + datastream_name: 'logs-enabled-no-retention', + dsl: { enabled: true, data_retention: undefined }, + indices: [{ index_name: '.ds-logs-2-000001', ilm_policy: undefined }], + }, + { + datastream_name: 'logs-disabled-no-lifecycle', + dsl: { enabled: false, data_retention: undefined }, + indices: [{ index_name: '.ds-logs-3-000001', ilm_policy: undefined }], + }, + { + datastream_name: 'logs-disabled-with-retention', + dsl: { enabled: false, data_retention: '7d' }, + indices: [{ index_name: '.ds-logs-4-000001', ilm_policy: undefined }], + }, + ]); + }); + + it('should handle null lifecycle object', async () => { + const mockResponse = { + data_streams: [ + { + name: 'logs-null-lifecycle', + lifecycle: null, + indices: [{ index_name: '.ds-logs-000001' }], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toEqual([ + { + datastream_name: 'logs-null-lifecycle', + dsl: { + enabled: false, + data_retention: undefined, + }, + indices: [{ index_name: '.ds-logs-000001', ilm_policy: undefined }], + }, + ]); + }); + + it('should handle null data_retention value', async () => { + const mockResponse = { + data_streams: [ + { + name: 'logs-null-retention', + lifecycle: { + enabled: true, + data_retention: null, + }, + indices: [{ index_name: '.ds-logs-000001' }], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toEqual([ + { + datastream_name: 'logs-null-retention', + dsl: { + enabled: true, + data_retention: null, + }, + indices: [{ index_name: '.ds-logs-000001', ilm_policy: undefined }], + }, + ]); + }); + + it.each([ + { description: 'hours', retention: '24h' }, + { description: 'minutes', retention: '1440m' }, + { description: 'mixed case', retention: '30D' }, + { description: 'with spaces', retention: '7 d' }, + { description: 'empty string', retention: '' }, + ])('should preserve retention format as-is: $description', async ({ retention }) => { + const mockResponse = { + data_streams: [ + { + name: 'logs-format-test', + lifecycle: { + enabled: true, + data_retention: retention, + }, + indices: [], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result[0].dsl?.data_retention).toBe(retention); + }); + + it('should handle datastreams with both DSL and ILM policy', async () => { + const mockResponse = { + data_streams: [ + { + name: 'logs-both-dsl-ilm', + lifecycle: { + enabled: true, + data_retention: '30d', + }, + indices: [ + { + index_name: '.ds-logs-000001', + ilm_policy: 'logs-policy', + }, + ], + }, + ], + }; + + (esClient.indices.getDataStream as jest.Mock).mockResolvedValue(mockResponse); + + const result = await receiver.getDataStreams(); + + expect(result).toEqual([ + { + datastream_name: 'logs-both-dsl-ilm', + dsl: { + enabled: true, + data_retention: '30d', + }, + indices: [ + { + index_name: '.ds-logs-000001', + ilm_policy: 'logs-policy', + }, + ], + }, + ]); + }); }); describe('getIndexTemplatesStats', () => {