From c1b3e6c2863329e85f078267498c4178a512a2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 29 Aug 2025 18:06:45 +0200 Subject: [PATCH] [Telemetry][Indices Metadata] Update index stats query (#233500) ## Summary Include per-primaries data in index stats query. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) - [ ] Review the [backport guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing) and apply applicable `backport:*` labels. (cherry picked from commit f6f24122daeb79e9e326fc823a0bde10bee61df8) --- .../server/lib/services/receiver.test.ts | 163 ++++++++++++++++++ .../server/lib/services/receiver.ts | 16 ++ 2 files changed, 179 insertions(+) diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts index 16f620efcf6ac..8bb3f60db1ef1 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts @@ -405,6 +405,19 @@ describe('Indices Metadata - MetadataReceiver', () => { store: { size_in_bytes: 1024000, }, + indexing: { + index_failed: 5, + index_failed_due_to_version_conflict: 2, + }, + }, + primaries: { + docs: { + count: 250, + deleted: 5, + }, + store: { + size_in_bytes: 512000, + }, }, }, }, @@ -426,9 +439,17 @@ describe('Indices Metadata - MetadataReceiver', () => { filter_path: [ 'indices.*.total.search.query_total', 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', 'indices.*.total.docs.deleted', 'indices.*.total.store.size_in_bytes', + + 'indices.*.primaries.docs.count', + 'indices.*.primaries.docs.deleted', + 'indices.*.primaries.store.size_in_bytes', + + 'indices.*.total.indexing.index_failed', + 'indices.*.total.indexing.index_failed_due_to_version_conflict', ], }); @@ -440,6 +461,11 @@ describe('Indices Metadata - MetadataReceiver', () => { docs_count: 500, docs_deleted: 10, docs_total_size_in_bytes: 1024000, + index_failed: 5, + index_failed_due_to_version_conflict: 2, + docs_count_primaries: 250, + docs_deleted_primaries: 5, + docs_total_size_in_bytes_primaries: 512000, }, ]); }); @@ -500,6 +526,143 @@ describe('Indices Metadata - MetadataReceiver', () => { await expect(iterator.next()).rejects.toThrow('Elasticsearch error'); expect(logger.error).toHaveBeenCalledWith('Error fetching indices stats', { error }); }); + + it('should handle response with missing primaries data', async () => { + const mockResponseWithoutPrimaries = { + indices: { + 'test-index-1': { + total: { + search: { + query_total: 100, + query_time_in_millis: 1000, + }, + docs: { + count: 500, + deleted: 10, + }, + store: { + size_in_bytes: 1024000, + }, + indexing: { + index_failed: 5, + index_failed_due_to_version_conflict: 2, + }, + }, + }, + }, + }; + + (esClient.indices.stats as jest.Mock).mockResolvedValue(mockResponseWithoutPrimaries); + + const results = []; + for await (const stat of receiver.getIndicesStats(['test-index-1'], 10)) { + results.push(stat); + } + + expect(results).toEqual([ + { + index_name: 'test-index-1', + query_total: 100, + query_time_in_millis: 1000, + docs_count: 500, + docs_deleted: 10, + docs_total_size_in_bytes: 1024000, + index_failed: 5, + index_failed_due_to_version_conflict: 2, + docs_count_primaries: undefined, + docs_deleted_primaries: undefined, + docs_total_size_in_bytes_primaries: undefined, + }, + ]); + }); + + it('should handle response with missing indexing failure data', async () => { + const mockResponseWithoutIndexingFailures = { + indices: { + 'test-index-1': { + total: { + search: { + query_total: 100, + query_time_in_millis: 1000, + }, + docs: { + count: 500, + deleted: 10, + }, + store: { + size_in_bytes: 1024000, + }, + }, + primaries: { + docs: { + count: 250, + deleted: 5, + }, + store: { + size_in_bytes: 512000, + }, + }, + }, + }, + }; + + (esClient.indices.stats as jest.Mock).mockResolvedValue(mockResponseWithoutIndexingFailures); + + const results = []; + for await (const stat of receiver.getIndicesStats(['test-index-1'], 10)) { + results.push(stat); + } + + expect(results).toEqual([ + { + index_name: 'test-index-1', + query_total: 100, + query_time_in_millis: 1000, + docs_count: 500, + docs_deleted: 10, + docs_total_size_in_bytes: 1024000, + index_failed: undefined, + index_failed_due_to_version_conflict: undefined, + docs_count_primaries: 250, + docs_deleted_primaries: 5, + docs_total_size_in_bytes_primaries: 512000, + }, + ]); + }); + + it('should continue processing when receiver methods return empty results', async () => { + const mockResponseMinimalData = { + indices: { + 'test-index-1': { + total: {}, + primaries: {}, + }, + }, + }; + + (esClient.indices.stats as jest.Mock).mockResolvedValue(mockResponseMinimalData); + + const results = []; + for await (const stat of receiver.getIndicesStats(['test-index-1'], 10)) { + results.push(stat); + } + + expect(results).toEqual([ + { + index_name: 'test-index-1', + query_total: undefined, + query_time_in_millis: undefined, + docs_count: undefined, + docs_deleted: undefined, + docs_total_size_in_bytes: undefined, + index_failed: undefined, + index_failed_due_to_version_conflict: undefined, + docs_count_primaries: undefined, + docs_deleted_primaries: undefined, + docs_total_size_in_bytes_primaries: undefined, + }, + ]); + }); }); describe('isIlmStatsAvailable', () => { diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts index 12bfddbca6405..2cc980c3ed901 100644 --- a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts @@ -121,9 +121,17 @@ export class MetadataReceiver { filter_path: [ 'indices.*.total.search.query_total', 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', 'indices.*.total.docs.deleted', 'indices.*.total.store.size_in_bytes', + + 'indices.*.primaries.docs.count', + 'indices.*.primaries.docs.deleted', + 'indices.*.primaries.store.size_in_bytes', + + 'indices.*.total.indexing.index_failed', + 'indices.*.total.indexing.index_failed_due_to_version_conflict', ], }; @@ -137,6 +145,14 @@ export class MetadataReceiver { docs_count: stats.total?.docs?.count, docs_deleted: stats.total?.docs?.deleted, docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + + index_failed: stats.total?.indexing?.index_failed, + index_failed_due_to_version_conflict: (stats.total?.indexing as any) + ?.index_failed_due_to_version_conflict, + + docs_count_primaries: stats.primaries?.docs?.count, + docs_deleted_primaries: stats.primaries?.docs?.deleted, + docs_total_size_in_bytes_primaries: stats.primaries?.store?.size_in_bytes, } as IndexStats; } } catch (error) {